| { |
| "metadata": { |
| "tool": "abliterlitics", |
| "version": "1.0.0", |
| "results_version": 1, |
| "comparison_name": "gemma4-e4b", |
| "variant": "coder3101", |
| "architecture": "gemma4", |
| "timestamp": "2026-06-27T11:41:07.560739+00:00" |
| }, |
| "results_version": 1, |
| "architecture": { |
| "family": "gemma4", |
| "layers": 42 |
| }, |
| "variant_a": "Coder3101 Heretic", |
| "variant_b": null, |
| "total_analyzed": 719, |
| "full_svd": false, |
| "tensor_results": [ |
| { |
| "key": "model.language_model.embed_tokens.weight", |
| "layer": null, |
| "tensor_type": "embed_tokens.weight", |
| "category": "embedding", |
| "shape": [ |
| 262144, |
| 2560 |
| ], |
| "numel": 671088640, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.embed_tokens_per_layer.weight", |
| "layer": null, |
| "tensor_type": "embed_tokens_per_layer.weight", |
| "category": "embedding", |
| "shape": [ |
| 262144, |
| 10752 |
| ], |
| "numel": 2818572288, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.input_layernorm.weight", |
| "layer": 0, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.layer_scalar", |
| "layer": 0, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.mlp.down_proj.weight", |
| "layer": 0, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.mlp.gate_proj.weight", |
| "layer": 0, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.mlp.up_proj.weight", |
| "layer": 0, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.per_layer_input_gate.weight", |
| "layer": 0, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.per_layer_projection.weight", |
| "layer": 0, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.post_attention_layernorm.weight", |
| "layer": 0, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.post_feedforward_layernorm.weight", |
| "layer": 0, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.post_per_layer_input_norm.weight", |
| "layer": 0, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.pre_feedforward_layernorm.weight", |
| "layer": 0, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.self_attn.k_norm.weight", |
| "layer": 0, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.self_attn.k_proj.weight", |
| "layer": 0, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.self_attn.o_proj.weight", |
| "layer": 0, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.self_attn.q_norm.weight", |
| "layer": 0, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.self_attn.q_proj.weight", |
| "layer": 0, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.0.self_attn.v_proj.weight", |
| "layer": 0, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.input_layernorm.weight", |
| "layer": 1, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.layer_scalar", |
| "layer": 1, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.mlp.down_proj.weight", |
| "layer": 1, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.mlp.gate_proj.weight", |
| "layer": 1, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.mlp.up_proj.weight", |
| "layer": 1, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.per_layer_input_gate.weight", |
| "layer": 1, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.per_layer_projection.weight", |
| "layer": 1, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.post_attention_layernorm.weight", |
| "layer": 1, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.post_feedforward_layernorm.weight", |
| "layer": 1, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.post_per_layer_input_norm.weight", |
| "layer": 1, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.pre_feedforward_layernorm.weight", |
| "layer": 1, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.self_attn.k_norm.weight", |
| "layer": 1, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.self_attn.k_proj.weight", |
| "layer": 1, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.self_attn.o_proj.weight", |
| "layer": 1, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.self_attn.q_norm.weight", |
| "layer": 1, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.self_attn.q_proj.weight", |
| "layer": 1, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.1.self_attn.v_proj.weight", |
| "layer": 1, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.input_layernorm.weight", |
| "layer": 10, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.layer_scalar", |
| "layer": 10, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.mlp.down_proj.weight", |
| "layer": 10, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.mlp.gate_proj.weight", |
| "layer": 10, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.mlp.up_proj.weight", |
| "layer": 10, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.per_layer_input_gate.weight", |
| "layer": 10, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.per_layer_projection.weight", |
| "layer": 10, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.post_attention_layernorm.weight", |
| "layer": 10, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.post_feedforward_layernorm.weight", |
| "layer": 10, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.post_per_layer_input_norm.weight", |
| "layer": 10, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.pre_feedforward_layernorm.weight", |
| "layer": 10, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.self_attn.k_norm.weight", |
| "layer": 10, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.self_attn.k_proj.weight", |
| "layer": 10, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.self_attn.o_proj.weight", |
| "layer": 10, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.self_attn.q_norm.weight", |
| "layer": 10, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.self_attn.q_proj.weight", |
| "layer": 10, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.10.self_attn.v_proj.weight", |
| "layer": 10, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.input_layernorm.weight", |
| "layer": 11, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.layer_scalar", |
| "layer": 11, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.mlp.down_proj.weight", |
| "layer": 11, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.mlp.gate_proj.weight", |
| "layer": 11, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.mlp.up_proj.weight", |
| "layer": 11, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.per_layer_input_gate.weight", |
| "layer": 11, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.per_layer_projection.weight", |
| "layer": 11, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.post_attention_layernorm.weight", |
| "layer": 11, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.post_feedforward_layernorm.weight", |
| "layer": 11, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.post_per_layer_input_norm.weight", |
| "layer": 11, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.pre_feedforward_layernorm.weight", |
| "layer": 11, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.self_attn.k_norm.weight", |
| "layer": 11, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.self_attn.k_proj.weight", |
| "layer": 11, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.self_attn.o_proj.weight", |
| "layer": 11, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.self_attn.q_norm.weight", |
| "layer": 11, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.self_attn.q_proj.weight", |
| "layer": 11, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.11.self_attn.v_proj.weight", |
| "layer": 11, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.input_layernorm.weight", |
| "layer": 12, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.layer_scalar", |
| "layer": 12, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.mlp.down_proj.weight", |
| "layer": 12, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.mlp.gate_proj.weight", |
| "layer": 12, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.mlp.up_proj.weight", |
| "layer": 12, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.per_layer_input_gate.weight", |
| "layer": 12, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.per_layer_projection.weight", |
| "layer": 12, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.post_attention_layernorm.weight", |
| "layer": 12, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.post_feedforward_layernorm.weight", |
| "layer": 12, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.post_per_layer_input_norm.weight", |
| "layer": 12, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.pre_feedforward_layernorm.weight", |
| "layer": 12, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.self_attn.k_norm.weight", |
| "layer": 12, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.self_attn.k_proj.weight", |
| "layer": 12, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.self_attn.o_proj.weight", |
| "layer": 12, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.self_attn.q_norm.weight", |
| "layer": 12, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.self_attn.q_proj.weight", |
| "layer": 12, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.12.self_attn.v_proj.weight", |
| "layer": 12, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.input_layernorm.weight", |
| "layer": 13, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.layer_scalar", |
| "layer": 13, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.mlp.down_proj.weight", |
| "layer": 13, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.mlp.gate_proj.weight", |
| "layer": 13, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.mlp.up_proj.weight", |
| "layer": 13, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.per_layer_input_gate.weight", |
| "layer": 13, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.per_layer_projection.weight", |
| "layer": 13, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.post_attention_layernorm.weight", |
| "layer": 13, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.post_feedforward_layernorm.weight", |
| "layer": 13, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.post_per_layer_input_norm.weight", |
| "layer": 13, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.pre_feedforward_layernorm.weight", |
| "layer": 13, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.self_attn.k_norm.weight", |
| "layer": 13, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.self_attn.k_proj.weight", |
| "layer": 13, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.self_attn.o_proj.weight", |
| "layer": 13, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.self_attn.q_norm.weight", |
| "layer": 13, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.self_attn.q_proj.weight", |
| "layer": 13, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.13.self_attn.v_proj.weight", |
| "layer": 13, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.input_layernorm.weight", |
| "layer": 14, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.layer_scalar", |
| "layer": 14, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.mlp.down_proj.weight", |
| "layer": 14, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.mlp.gate_proj.weight", |
| "layer": 14, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.mlp.up_proj.weight", |
| "layer": 14, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.per_layer_input_gate.weight", |
| "layer": 14, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.per_layer_projection.weight", |
| "layer": 14, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.post_attention_layernorm.weight", |
| "layer": 14, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.post_feedforward_layernorm.weight", |
| "layer": 14, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.post_per_layer_input_norm.weight", |
| "layer": 14, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.pre_feedforward_layernorm.weight", |
| "layer": 14, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.self_attn.k_norm.weight", |
| "layer": 14, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.self_attn.k_proj.weight", |
| "layer": 14, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.self_attn.o_proj.weight", |
| "layer": 14, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.self_attn.q_norm.weight", |
| "layer": 14, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.self_attn.q_proj.weight", |
| "layer": 14, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.14.self_attn.v_proj.weight", |
| "layer": 14, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.input_layernorm.weight", |
| "layer": 15, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.layer_scalar", |
| "layer": 15, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.mlp.down_proj.weight", |
| "layer": 15, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.mlp.gate_proj.weight", |
| "layer": 15, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.mlp.up_proj.weight", |
| "layer": 15, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.per_layer_input_gate.weight", |
| "layer": 15, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.per_layer_projection.weight", |
| "layer": 15, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.post_attention_layernorm.weight", |
| "layer": 15, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.post_feedforward_layernorm.weight", |
| "layer": 15, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.post_per_layer_input_norm.weight", |
| "layer": 15, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.pre_feedforward_layernorm.weight", |
| "layer": 15, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.self_attn.k_norm.weight", |
| "layer": 15, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.self_attn.k_proj.weight", |
| "layer": 15, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.self_attn.o_proj.weight", |
| "layer": 15, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.self_attn.q_norm.weight", |
| "layer": 15, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.self_attn.q_proj.weight", |
| "layer": 15, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.15.self_attn.v_proj.weight", |
| "layer": 15, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.input_layernorm.weight", |
| "layer": 16, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.layer_scalar", |
| "layer": 16, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.mlp.down_proj.weight", |
| "layer": 16, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.mlp.gate_proj.weight", |
| "layer": 16, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.mlp.up_proj.weight", |
| "layer": 16, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.per_layer_input_gate.weight", |
| "layer": 16, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.per_layer_projection.weight", |
| "layer": 16, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.post_attention_layernorm.weight", |
| "layer": 16, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.post_feedforward_layernorm.weight", |
| "layer": 16, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.post_per_layer_input_norm.weight", |
| "layer": 16, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.pre_feedforward_layernorm.weight", |
| "layer": 16, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.self_attn.k_norm.weight", |
| "layer": 16, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.self_attn.k_proj.weight", |
| "layer": 16, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.self_attn.o_proj.weight", |
| "layer": 16, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.self_attn.q_norm.weight", |
| "layer": 16, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.self_attn.q_proj.weight", |
| "layer": 16, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.16.self_attn.v_proj.weight", |
| "layer": 16, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.input_layernorm.weight", |
| "layer": 17, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.layer_scalar", |
| "layer": 17, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.mlp.down_proj.weight", |
| "layer": 17, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.mlp.gate_proj.weight", |
| "layer": 17, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.mlp.up_proj.weight", |
| "layer": 17, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.per_layer_input_gate.weight", |
| "layer": 17, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.per_layer_projection.weight", |
| "layer": 17, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.post_attention_layernorm.weight", |
| "layer": 17, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.post_feedforward_layernorm.weight", |
| "layer": 17, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.post_per_layer_input_norm.weight", |
| "layer": 17, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.pre_feedforward_layernorm.weight", |
| "layer": 17, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.self_attn.k_norm.weight", |
| "layer": 17, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.self_attn.k_proj.weight", |
| "layer": 17, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.self_attn.o_proj.weight", |
| "layer": 17, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.self_attn.q_norm.weight", |
| "layer": 17, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.self_attn.q_proj.weight", |
| "layer": 17, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.17.self_attn.v_proj.weight", |
| "layer": 17, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.input_layernorm.weight", |
| "layer": 18, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.layer_scalar", |
| "layer": 18, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.mlp.down_proj.weight", |
| "layer": 18, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.mlp.gate_proj.weight", |
| "layer": 18, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.mlp.up_proj.weight", |
| "layer": 18, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.per_layer_input_gate.weight", |
| "layer": 18, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.per_layer_projection.weight", |
| "layer": 18, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.post_attention_layernorm.weight", |
| "layer": 18, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.post_feedforward_layernorm.weight", |
| "layer": 18, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.post_per_layer_input_norm.weight", |
| "layer": 18, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.pre_feedforward_layernorm.weight", |
| "layer": 18, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.self_attn.k_norm.weight", |
| "layer": 18, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.self_attn.k_proj.weight", |
| "layer": 18, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.self_attn.o_proj.weight", |
| "layer": 18, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.self_attn.q_norm.weight", |
| "layer": 18, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.self_attn.q_proj.weight", |
| "layer": 18, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.18.self_attn.v_proj.weight", |
| "layer": 18, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.input_layernorm.weight", |
| "layer": 19, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.layer_scalar", |
| "layer": 19, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.mlp.down_proj.weight", |
| "layer": 19, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.mlp.gate_proj.weight", |
| "layer": 19, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.mlp.up_proj.weight", |
| "layer": 19, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.per_layer_input_gate.weight", |
| "layer": 19, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.per_layer_projection.weight", |
| "layer": 19, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.post_attention_layernorm.weight", |
| "layer": 19, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.post_feedforward_layernorm.weight", |
| "layer": 19, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.post_per_layer_input_norm.weight", |
| "layer": 19, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.pre_feedforward_layernorm.weight", |
| "layer": 19, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.self_attn.k_norm.weight", |
| "layer": 19, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.self_attn.k_proj.weight", |
| "layer": 19, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.self_attn.o_proj.weight", |
| "layer": 19, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.self_attn.q_norm.weight", |
| "layer": 19, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.self_attn.q_proj.weight", |
| "layer": 19, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.19.self_attn.v_proj.weight", |
| "layer": 19, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.input_layernorm.weight", |
| "layer": 2, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.layer_scalar", |
| "layer": 2, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.mlp.down_proj.weight", |
| "layer": 2, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.mlp.gate_proj.weight", |
| "layer": 2, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.mlp.up_proj.weight", |
| "layer": 2, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.per_layer_input_gate.weight", |
| "layer": 2, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.per_layer_projection.weight", |
| "layer": 2, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.post_attention_layernorm.weight", |
| "layer": 2, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.post_feedforward_layernorm.weight", |
| "layer": 2, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.post_per_layer_input_norm.weight", |
| "layer": 2, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.pre_feedforward_layernorm.weight", |
| "layer": 2, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.self_attn.k_norm.weight", |
| "layer": 2, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.self_attn.k_proj.weight", |
| "layer": 2, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.self_attn.o_proj.weight", |
| "layer": 2, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.self_attn.q_norm.weight", |
| "layer": 2, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.self_attn.q_proj.weight", |
| "layer": 2, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.2.self_attn.v_proj.weight", |
| "layer": 2, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.input_layernorm.weight", |
| "layer": 20, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.layer_scalar", |
| "layer": 20, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.mlp.down_proj.weight", |
| "layer": 20, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.mlp.gate_proj.weight", |
| "layer": 20, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.mlp.up_proj.weight", |
| "layer": 20, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.per_layer_input_gate.weight", |
| "layer": 20, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.per_layer_projection.weight", |
| "layer": 20, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.post_attention_layernorm.weight", |
| "layer": 20, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.post_feedforward_layernorm.weight", |
| "layer": 20, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.post_per_layer_input_norm.weight", |
| "layer": 20, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.pre_feedforward_layernorm.weight", |
| "layer": 20, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.self_attn.k_norm.weight", |
| "layer": 20, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.self_attn.k_proj.weight", |
| "layer": 20, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.self_attn.o_proj.weight", |
| "layer": 20, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.self_attn.q_norm.weight", |
| "layer": 20, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.self_attn.q_proj.weight", |
| "layer": 20, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.20.self_attn.v_proj.weight", |
| "layer": 20, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.input_layernorm.weight", |
| "layer": 21, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.layer_scalar", |
| "layer": 21, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.mlp.down_proj.weight", |
| "layer": 21, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.mlp.gate_proj.weight", |
| "layer": 21, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.mlp.up_proj.weight", |
| "layer": 21, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.per_layer_input_gate.weight", |
| "layer": 21, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.per_layer_projection.weight", |
| "layer": 21, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.post_attention_layernorm.weight", |
| "layer": 21, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.post_feedforward_layernorm.weight", |
| "layer": 21, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.post_per_layer_input_norm.weight", |
| "layer": 21, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.pre_feedforward_layernorm.weight", |
| "layer": 21, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.self_attn.k_norm.weight", |
| "layer": 21, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.self_attn.k_proj.weight", |
| "layer": 21, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.self_attn.o_proj.weight", |
| "layer": 21, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 1.4930545091629028, |
| "mean_abs": 0.0003818950499407947, |
| "max_abs": 0.02294921875, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 7.3843302726745605, |
| "top_singular_values": [ |
| 1.4204989671707153, |
| 0.19236665964126587, |
| 0.15514016151428223, |
| 0.13964632153511047, |
| 0.08826004713773727, |
| 0.07175162434577942, |
| 0.058954138308763504, |
| 0.05393417552113533, |
| 0.05035132169723511, |
| 0.045089323073625565, |
| 0.039022207260131836, |
| 0.03411183878779411, |
| 0.030919011682271957, |
| 0.029600005596876144, |
| 0.02805229462683201, |
| 0.027135420590639114, |
| 0.026436995714902878, |
| 0.024411456659436226, |
| 0.0231474656611681, |
| 0.02058911882340908 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 90.51707458496094, |
| "energy_top5_pct": 94.48098754882812, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 95.57544177605156 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.self_attn.q_norm.weight", |
| "layer": 21, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.self_attn.q_proj.weight", |
| "layer": 21, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.21.self_attn.v_proj.weight", |
| "layer": 21, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.input_layernorm.weight", |
| "layer": 22, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.layer_scalar", |
| "layer": 22, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.mlp.down_proj.weight", |
| "layer": 22, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.mlp.gate_proj.weight", |
| "layer": 22, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.mlp.up_proj.weight", |
| "layer": 22, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.per_layer_input_gate.weight", |
| "layer": 22, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.per_layer_projection.weight", |
| "layer": 22, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.post_attention_layernorm.weight", |
| "layer": 22, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.post_feedforward_layernorm.weight", |
| "layer": 22, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.post_per_layer_input_norm.weight", |
| "layer": 22, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.pre_feedforward_layernorm.weight", |
| "layer": 22, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.self_attn.k_norm.weight", |
| "layer": 22, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.self_attn.k_proj.weight", |
| "layer": 22, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.self_attn.o_proj.weight", |
| "layer": 22, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.118997812271118, |
| "mean_abs": 0.00046930459211580455, |
| "max_abs": 0.01806640625, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 8.39266586303711, |
| "top_singular_values": [ |
| 2.067248821258545, |
| 0.2463161200284958, |
| 0.158758282661438, |
| 0.13891978561878204, |
| 0.08195827156305313, |
| 0.052530672401189804, |
| 0.05087503045797348, |
| 0.047861497849226, |
| 0.0381326824426651, |
| 0.03511792793869972, |
| 0.03160722181200981, |
| 0.029758378863334656, |
| 0.02595704048871994, |
| 0.02530757337808609, |
| 0.02401380054652691, |
| 0.021632060408592224, |
| 0.02099302038550377, |
| 0.020074840635061264, |
| 0.01966053806245327, |
| 0.018524209037423134 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 95.17534637451172, |
| "energy_top5_pct": 97.66728210449219, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 98.02679262776408 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.self_attn.q_norm.weight", |
| "layer": 22, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.self_attn.q_proj.weight", |
| "layer": 22, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.22.self_attn.v_proj.weight", |
| "layer": 22, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.input_layernorm.weight", |
| "layer": 23, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.layer_scalar", |
| "layer": 23, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.mlp.down_proj.weight", |
| "layer": 23, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.mlp.gate_proj.weight", |
| "layer": 23, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.mlp.up_proj.weight", |
| "layer": 23, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.per_layer_input_gate.weight", |
| "layer": 23, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.per_layer_projection.weight", |
| "layer": 23, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.post_attention_layernorm.weight", |
| "layer": 23, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.post_feedforward_layernorm.weight", |
| "layer": 23, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.post_per_layer_input_norm.weight", |
| "layer": 23, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.pre_feedforward_layernorm.weight", |
| "layer": 23, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.self_attn.k_norm.weight", |
| "layer": 23, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.self_attn.k_proj.weight", |
| "layer": 23, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.self_attn.o_proj.weight", |
| "layer": 23, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 1.7929438352584839, |
| "mean_abs": 0.00027919490821659565, |
| "max_abs": 0.02353668212890625, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 19.305932998657227, |
| "top_singular_values": [ |
| 1.7475371360778809, |
| 0.09051813930273056, |
| 0.06224147230386734, |
| 0.05410454049706459, |
| 0.0483347550034523, |
| 0.046585436910390854, |
| 0.04311037063598633, |
| 0.04151644930243492, |
| 0.04000209644436836, |
| 0.03959033638238907, |
| 0.027680905535817146, |
| 0.024704452604055405, |
| 0.022998850792646408, |
| 0.020372796803712845, |
| 0.019744373857975006, |
| 0.018514418974518776, |
| 0.018368231132626534, |
| 0.018016338348388672, |
| 0.017500482499599457, |
| 0.016903802752494812 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 94.99909210205078, |
| "energy_top5_pct": 95.53822326660156, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 95.94967888495606 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.self_attn.q_norm.weight", |
| "layer": 23, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.self_attn.q_proj.weight", |
| "layer": 23, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.23.self_attn.v_proj.weight", |
| "layer": 23, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.input_layernorm.weight", |
| "layer": 24, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.layer_scalar", |
| "layer": 24, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.mlp.down_proj.weight", |
| "layer": 24, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.mlp.gate_proj.weight", |
| "layer": 24, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.mlp.up_proj.weight", |
| "layer": 24, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.per_layer_input_gate.weight", |
| "layer": 24, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.per_layer_projection.weight", |
| "layer": 24, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.post_attention_layernorm.weight", |
| "layer": 24, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.post_feedforward_layernorm.weight", |
| "layer": 24, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.post_per_layer_input_norm.weight", |
| "layer": 24, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.pre_feedforward_layernorm.weight", |
| "layer": 24, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.self_attn.k_norm.weight", |
| "layer": 24, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.self_attn.k_proj.weight", |
| "layer": 24, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.self_attn.o_proj.weight", |
| "layer": 24, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.4170305728912354, |
| "mean_abs": 0.0005704322829842567, |
| "max_abs": 0.02728271484375, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 9.989450454711914, |
| "top_singular_values": [ |
| 2.37009334564209, |
| 0.237259641289711, |
| 0.13717225193977356, |
| 0.12748974561691284, |
| 0.10416634380817413, |
| 0.05667352303862572, |
| 0.0530715249478817, |
| 0.046780064702034, |
| 0.04022771492600441, |
| 0.037861283868551254, |
| 0.03289154917001724, |
| 0.029343046247959137, |
| 0.026425033807754517, |
| 0.02574390545487404, |
| 0.025480519980192184, |
| 0.022529203444719315, |
| 0.02218289114534855, |
| 0.020708557218313217, |
| 0.017748361453413963, |
| 0.016974152997136116 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 96.15383911132812, |
| "energy_top5_pct": 97.9034423828125, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 98.19877865866326 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.self_attn.q_norm.weight", |
| "layer": 24, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.self_attn.q_proj.weight", |
| "layer": 24, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.24.self_attn.v_proj.weight", |
| "layer": 24, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.input_layernorm.weight", |
| "layer": 25, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.layer_scalar", |
| "layer": 25, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.mlp.down_proj.weight", |
| "layer": 25, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.mlp.gate_proj.weight", |
| "layer": 25, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.mlp.up_proj.weight", |
| "layer": 25, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.per_layer_input_gate.weight", |
| "layer": 25, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.per_layer_projection.weight", |
| "layer": 25, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.post_attention_layernorm.weight", |
| "layer": 25, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.post_feedforward_layernorm.weight", |
| "layer": 25, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.post_per_layer_input_norm.weight", |
| "layer": 25, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.pre_feedforward_layernorm.weight", |
| "layer": 25, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.self_attn.k_norm.weight", |
| "layer": 25, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.self_attn.k_proj.weight", |
| "layer": 25, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.self_attn.o_proj.weight", |
| "layer": 25, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.5245981216430664, |
| "mean_abs": 0.0005986131145618856, |
| "max_abs": 0.02978515625, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 8.95284652709961, |
| "top_singular_values": [ |
| 2.4384546279907227, |
| 0.27236640453338623, |
| 0.2362680733203888, |
| 0.18323780596256256, |
| 0.13827981054782867, |
| 0.0857291892170906, |
| 0.08388141542673111, |
| 0.07315989583730698, |
| 0.06196226179599762, |
| 0.04916068911552429, |
| 0.04583769664168358, |
| 0.04261838644742966, |
| 0.03968231379985809, |
| 0.03883890435099602, |
| 0.03816549852490425, |
| 0.03532534837722778, |
| 0.03234706446528435, |
| 0.03084811381995678, |
| 0.025621948763728142, |
| 0.0222295131534338 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 93.29209899902344, |
| "energy_top5_pct": 96.15866088867188, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 96.76818026058889 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.self_attn.q_norm.weight", |
| "layer": 25, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.self_attn.q_proj.weight", |
| "layer": 25, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.25.self_attn.v_proj.weight", |
| "layer": 25, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.input_layernorm.weight", |
| "layer": 26, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.layer_scalar", |
| "layer": 26, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.mlp.down_proj.weight", |
| "layer": 26, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.mlp.gate_proj.weight", |
| "layer": 26, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.mlp.up_proj.weight", |
| "layer": 26, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.per_layer_input_gate.weight", |
| "layer": 26, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.per_layer_projection.weight", |
| "layer": 26, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.post_attention_layernorm.weight", |
| "layer": 26, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.post_feedforward_layernorm.weight", |
| "layer": 26, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.post_per_layer_input_norm.weight", |
| "layer": 26, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.pre_feedforward_layernorm.weight", |
| "layer": 26, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.self_attn.k_norm.weight", |
| "layer": 26, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.self_attn.k_proj.weight", |
| "layer": 26, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.self_attn.o_proj.weight", |
| "layer": 26, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 1.9971189498901367, |
| "mean_abs": 0.0005002947291359305, |
| "max_abs": 0.033203125, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 12.312246322631836, |
| "top_singular_values": [ |
| 1.9569448232650757, |
| 0.15894295275211334, |
| 0.13918840885162354, |
| 0.08817492425441742, |
| 0.05451170355081558, |
| 0.039015352725982666, |
| 0.03478053957223892, |
| 0.03407804295420647, |
| 0.031064607203006744, |
| 0.02600487507879734, |
| 0.025158699601888657, |
| 0.022869804874062538, |
| 0.020230833441019058, |
| 0.019702086225152016, |
| 0.018938440829515457, |
| 0.01850656419992447, |
| 0.016695313155651093, |
| 0.015916984528303146, |
| 0.015307564288377762, |
| 0.014988488517701626 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 96.01725006103516, |
| "energy_top5_pct": 97.40581512451172, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.63598644029105 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.self_attn.q_norm.weight", |
| "layer": 26, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.self_attn.q_proj.weight", |
| "layer": 26, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.26.self_attn.v_proj.weight", |
| "layer": 26, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.input_layernorm.weight", |
| "layer": 27, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.layer_scalar", |
| "layer": 27, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.mlp.down_proj.weight", |
| "layer": 27, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.mlp.gate_proj.weight", |
| "layer": 27, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.mlp.up_proj.weight", |
| "layer": 27, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.per_layer_input_gate.weight", |
| "layer": 27, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.per_layer_projection.weight", |
| "layer": 27, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.post_attention_layernorm.weight", |
| "layer": 27, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.post_feedforward_layernorm.weight", |
| "layer": 27, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.post_per_layer_input_norm.weight", |
| "layer": 27, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.pre_feedforward_layernorm.weight", |
| "layer": 27, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.self_attn.k_norm.weight", |
| "layer": 27, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.self_attn.k_proj.weight", |
| "layer": 27, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.self_attn.o_proj.weight", |
| "layer": 27, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.11468243598938, |
| "mean_abs": 0.0004930722643621266, |
| "max_abs": 0.05517578125, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 8.617884635925293, |
| "top_singular_values": [ |
| 2.0418784618377686, |
| 0.23693500459194183, |
| 0.19846414029598236, |
| 0.1367007941007614, |
| 0.1194392666220665, |
| 0.08839662373065948, |
| 0.06784648448228836, |
| 0.06431671977043152, |
| 0.051705218851566315, |
| 0.04597106948494911, |
| 0.04462001472711563, |
| 0.04143412783741951, |
| 0.0385197214782238, |
| 0.03604878485202789, |
| 0.032358165830373764, |
| 0.03142313286662102, |
| 0.029791172593832016, |
| 0.02814287506043911, |
| 0.026523467153310776, |
| 0.025685930624604225 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 93.23295593261719, |
| "energy_top5_pct": 96.10599517822266, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 96.84190601542957 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.self_attn.q_norm.weight", |
| "layer": 27, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.self_attn.q_proj.weight", |
| "layer": 27, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.27.self_attn.v_proj.weight", |
| "layer": 27, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.input_layernorm.weight", |
| "layer": 28, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.layer_scalar", |
| "layer": 28, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.mlp.down_proj.weight", |
| "layer": 28, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.mlp.gate_proj.weight", |
| "layer": 28, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.mlp.up_proj.weight", |
| "layer": 28, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.per_layer_input_gate.weight", |
| "layer": 28, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.per_layer_projection.weight", |
| "layer": 28, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.post_attention_layernorm.weight", |
| "layer": 28, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.post_feedforward_layernorm.weight", |
| "layer": 28, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.post_per_layer_input_norm.weight", |
| "layer": 28, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.pre_feedforward_layernorm.weight", |
| "layer": 28, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.self_attn.k_norm.weight", |
| "layer": 28, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.self_attn.k_proj.weight", |
| "layer": 28, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.self_attn.o_proj.weight", |
| "layer": 28, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 1.695213794708252, |
| "mean_abs": 0.0004223079595249146, |
| "max_abs": 0.0400390625, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 13.25359058380127, |
| "top_singular_values": [ |
| 1.6600357294082642, |
| 0.12525177001953125, |
| 0.11073849350214005, |
| 0.0769524872303009, |
| 0.05475915968418121, |
| 0.041293755173683167, |
| 0.03677133843302727, |
| 0.033847708255052567, |
| 0.03003549575805664, |
| 0.027127405628561974, |
| 0.024418970569968224, |
| 0.023372987285256386, |
| 0.02250019647181034, |
| 0.020725134760141373, |
| 0.01728501357138157, |
| 0.015278040431439877, |
| 0.014728794805705547, |
| 0.01393070723861456, |
| 0.01329854130744934, |
| 0.013050783425569534 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 95.89278411865234, |
| "energy_top5_pct": 97.17582702636719, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.4963013358143 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.self_attn.q_norm.weight", |
| "layer": 28, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.self_attn.q_proj.weight", |
| "layer": 28, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.28.self_attn.v_proj.weight", |
| "layer": 28, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.input_layernorm.weight", |
| "layer": 29, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.layer_scalar", |
| "layer": 29, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.mlp.down_proj.weight", |
| "layer": 29, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.mlp.gate_proj.weight", |
| "layer": 29, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.mlp.up_proj.weight", |
| "layer": 29, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.per_layer_input_gate.weight", |
| "layer": 29, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.per_layer_projection.weight", |
| "layer": 29, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.post_attention_layernorm.weight", |
| "layer": 29, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.post_feedforward_layernorm.weight", |
| "layer": 29, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.post_per_layer_input_norm.weight", |
| "layer": 29, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.pre_feedforward_layernorm.weight", |
| "layer": 29, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.self_attn.k_norm.weight", |
| "layer": 29, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.self_attn.k_proj.weight", |
| "layer": 29, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.self_attn.o_proj.weight", |
| "layer": 29, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.5899410247802734, |
| "mean_abs": 0.0003514360578265041, |
| "max_abs": 0.10888671875, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 8.869449615478516, |
| "top_singular_values": [ |
| 2.5084893703460693, |
| 0.2828235626220703, |
| 0.2286226749420166, |
| 0.15444619953632355, |
| 0.11137975752353668, |
| 0.10735288262367249, |
| 0.09603819996118546, |
| 0.08677905797958374, |
| 0.07769200950860977, |
| 0.0617918036878109, |
| 0.05511785298585892, |
| 0.051078569144010544, |
| 0.04562080651521683, |
| 0.04329347610473633, |
| 0.035440199077129364, |
| 0.03502722829580307, |
| 0.02851518616080284, |
| 0.027447577565908432, |
| 0.0247665885835886, |
| 0.02380175143480301 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 93.8090591430664, |
| "energy_top5_pct": 96.3213119506836, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.11091896362257 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.self_attn.q_norm.weight", |
| "layer": 29, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.self_attn.q_proj.weight", |
| "layer": 29, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.29.self_attn.v_proj.weight", |
| "layer": 29, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.input_layernorm.weight", |
| "layer": 3, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.layer_scalar", |
| "layer": 3, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.mlp.down_proj.weight", |
| "layer": 3, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.mlp.gate_proj.weight", |
| "layer": 3, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.mlp.up_proj.weight", |
| "layer": 3, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.per_layer_input_gate.weight", |
| "layer": 3, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.per_layer_projection.weight", |
| "layer": 3, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.post_attention_layernorm.weight", |
| "layer": 3, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.post_feedforward_layernorm.weight", |
| "layer": 3, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.post_per_layer_input_norm.weight", |
| "layer": 3, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.pre_feedforward_layernorm.weight", |
| "layer": 3, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.self_attn.k_norm.weight", |
| "layer": 3, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.self_attn.k_proj.weight", |
| "layer": 3, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.self_attn.o_proj.weight", |
| "layer": 3, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.self_attn.q_norm.weight", |
| "layer": 3, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.self_attn.q_proj.weight", |
| "layer": 3, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.3.self_attn.v_proj.weight", |
| "layer": 3, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.input_layernorm.weight", |
| "layer": 30, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.layer_scalar", |
| "layer": 30, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.mlp.down_proj.weight", |
| "layer": 30, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.mlp.gate_proj.weight", |
| "layer": 30, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.mlp.up_proj.weight", |
| "layer": 30, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.per_layer_input_gate.weight", |
| "layer": 30, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.per_layer_projection.weight", |
| "layer": 30, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.post_attention_layernorm.weight", |
| "layer": 30, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.post_feedforward_layernorm.weight", |
| "layer": 30, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.post_per_layer_input_norm.weight", |
| "layer": 30, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.pre_feedforward_layernorm.weight", |
| "layer": 30, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.self_attn.k_norm.weight", |
| "layer": 30, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.self_attn.k_proj.weight", |
| "layer": 30, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.self_attn.o_proj.weight", |
| "layer": 30, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.3874661922454834, |
| "mean_abs": 0.000554397760424763, |
| "max_abs": 0.091796875, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 10.19024658203125, |
| "top_singular_values": [ |
| 2.3360595703125, |
| 0.2292446494102478, |
| 0.20963115990161896, |
| 0.11340680718421936, |
| 0.09057898074388504, |
| 0.08667786419391632, |
| 0.06674244999885559, |
| 0.04822060465812683, |
| 0.047059621661901474, |
| 0.04502301290631294, |
| 0.03963535279035568, |
| 0.03526109457015991, |
| 0.03291178122162819, |
| 0.027943558990955353, |
| 0.024353066459298134, |
| 0.022593993693590164, |
| 0.019135989248752594, |
| 0.01672523282468319, |
| 0.01665540598332882, |
| 0.015229024924337864 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 95.73998260498047, |
| "energy_top5_pct": 97.8025131225586, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 98.24939087731835 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.self_attn.q_norm.weight", |
| "layer": 30, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.self_attn.q_proj.weight", |
| "layer": 30, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.30.self_attn.v_proj.weight", |
| "layer": 30, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.input_layernorm.weight", |
| "layer": 31, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.layer_scalar", |
| "layer": 31, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.mlp.down_proj.weight", |
| "layer": 31, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.mlp.gate_proj.weight", |
| "layer": 31, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.mlp.up_proj.weight", |
| "layer": 31, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.per_layer_input_gate.weight", |
| "layer": 31, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.per_layer_projection.weight", |
| "layer": 31, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.post_attention_layernorm.weight", |
| "layer": 31, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.post_feedforward_layernorm.weight", |
| "layer": 31, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.post_per_layer_input_norm.weight", |
| "layer": 31, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.pre_feedforward_layernorm.weight", |
| "layer": 31, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.self_attn.k_norm.weight", |
| "layer": 31, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.self_attn.k_proj.weight", |
| "layer": 31, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.self_attn.o_proj.weight", |
| "layer": 31, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 3.4129393100738525, |
| "mean_abs": 0.0008527684840373695, |
| "max_abs": 0.05810546875, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 8.841360092163086, |
| "top_singular_values": [ |
| 3.3117082118988037, |
| 0.37457001209259033, |
| 0.29349029064178467, |
| 0.21826109290122986, |
| 0.17263051867485046, |
| 0.14229252934455872, |
| 0.09088650345802307, |
| 0.09014778584241867, |
| 0.07568318396806717, |
| 0.06984843313694, |
| 0.06161968410015106, |
| 0.0553426630795002, |
| 0.05290858447551727, |
| 0.04778434336185455, |
| 0.04178532212972641, |
| 0.03881525993347168, |
| 0.03633265569806099, |
| 0.03391285985708237, |
| 0.033349424600601196, |
| 0.029898710548877716 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 94.1557846069336, |
| "energy_top5_pct": 96.76460266113281, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.33903593752127 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.self_attn.q_norm.weight", |
| "layer": 31, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.self_attn.q_proj.weight", |
| "layer": 31, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.31.self_attn.v_proj.weight", |
| "layer": 31, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.input_layernorm.weight", |
| "layer": 32, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.layer_scalar", |
| "layer": 32, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.mlp.down_proj.weight", |
| "layer": 32, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.mlp.gate_proj.weight", |
| "layer": 32, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.mlp.up_proj.weight", |
| "layer": 32, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.per_layer_input_gate.weight", |
| "layer": 32, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.per_layer_projection.weight", |
| "layer": 32, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.post_attention_layernorm.weight", |
| "layer": 32, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.post_feedforward_layernorm.weight", |
| "layer": 32, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.post_per_layer_input_norm.weight", |
| "layer": 32, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.pre_feedforward_layernorm.weight", |
| "layer": 32, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.self_attn.k_norm.weight", |
| "layer": 32, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.self_attn.k_proj.weight", |
| "layer": 32, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.self_attn.o_proj.weight", |
| "layer": 32, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.291661262512207, |
| "mean_abs": 0.0005121523863635957, |
| "max_abs": 0.06787109375, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 6.882842063903809, |
| "top_singular_values": [ |
| 2.2128069400787354, |
| 0.3214960992336273, |
| 0.171408548951149, |
| 0.13484223186969757, |
| 0.11916665732860565, |
| 0.07949104905128479, |
| 0.06744391471147537, |
| 0.06554393470287323, |
| 0.054335277527570724, |
| 0.050705235451459885, |
| 0.05035466328263283, |
| 0.042536571621894836, |
| 0.039378125220537186, |
| 0.03533206507563591, |
| 0.031739190220832825, |
| 0.026621881872415543, |
| 0.024814872071146965, |
| 0.02347995527088642, |
| 0.021856410428881645, |
| 0.02044764533638954 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 93.2365493774414, |
| "energy_top5_pct": 96.3807373046875, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 96.98262303197241 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.self_attn.q_norm.weight", |
| "layer": 32, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.self_attn.q_proj.weight", |
| "layer": 32, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.32.self_attn.v_proj.weight", |
| "layer": 32, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.input_layernorm.weight", |
| "layer": 33, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.layer_scalar", |
| "layer": 33, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.mlp.down_proj.weight", |
| "layer": 33, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.mlp.gate_proj.weight", |
| "layer": 33, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.mlp.up_proj.weight", |
| "layer": 33, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.per_layer_input_gate.weight", |
| "layer": 33, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.per_layer_projection.weight", |
| "layer": 33, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.post_attention_layernorm.weight", |
| "layer": 33, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.post_feedforward_layernorm.weight", |
| "layer": 33, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.post_per_layer_input_norm.weight", |
| "layer": 33, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.pre_feedforward_layernorm.weight", |
| "layer": 33, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.self_attn.k_norm.weight", |
| "layer": 33, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.self_attn.k_proj.weight", |
| "layer": 33, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.self_attn.o_proj.weight", |
| "layer": 33, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.1235435009002686, |
| "mean_abs": 0.00046057128929533064, |
| "max_abs": 0.1123046875, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 9.143207550048828, |
| "top_singular_values": [ |
| 2.071141242980957, |
| 0.22652238607406616, |
| 0.13214591145515442, |
| 0.11728398501873016, |
| 0.08757036179304123, |
| 0.07215940952301025, |
| 0.06588569283485413, |
| 0.04756465554237366, |
| 0.04391246289014816, |
| 0.03922247514128685, |
| 0.036893170326948166, |
| 0.03370874002575874, |
| 0.030978217720985413, |
| 0.028699664399027824, |
| 0.02399620972573757, |
| 0.022257834672927856, |
| 0.02043365128338337, |
| 0.019933419302105904, |
| 0.019079888239502907, |
| 0.01754171960055828 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 95.12553405761719, |
| "energy_top5_pct": 97.12576293945312, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.61620573095601 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.self_attn.q_norm.weight", |
| "layer": 33, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.self_attn.q_proj.weight", |
| "layer": 33, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.33.self_attn.v_proj.weight", |
| "layer": 33, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.input_layernorm.weight", |
| "layer": 34, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.layer_scalar", |
| "layer": 34, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.mlp.down_proj.weight", |
| "layer": 34, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.mlp.gate_proj.weight", |
| "layer": 34, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.mlp.up_proj.weight", |
| "layer": 34, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.per_layer_input_gate.weight", |
| "layer": 34, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.per_layer_projection.weight", |
| "layer": 34, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.post_attention_layernorm.weight", |
| "layer": 34, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.post_feedforward_layernorm.weight", |
| "layer": 34, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.post_per_layer_input_norm.weight", |
| "layer": 34, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.pre_feedforward_layernorm.weight", |
| "layer": 34, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.self_attn.k_norm.weight", |
| "layer": 34, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.self_attn.k_proj.weight", |
| "layer": 34, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.self_attn.o_proj.weight", |
| "layer": 34, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.3655383586883545, |
| "mean_abs": 0.0005669129313901067, |
| "max_abs": 0.08203125, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 8.495016098022461, |
| "top_singular_values": [ |
| 2.2968482971191406, |
| 0.27037596702575684, |
| 0.18390822410583496, |
| 0.15070033073425293, |
| 0.12413713335990906, |
| 0.10499260574579239, |
| 0.06928076595067978, |
| 0.05840103700757027, |
| 0.055807001888751984, |
| 0.0547519214451313, |
| 0.049279727041721344, |
| 0.047826893627643585, |
| 0.04388877749443054, |
| 0.03862108662724495, |
| 0.03491431847214699, |
| 0.03198877349495888, |
| 0.029873991385102272, |
| 0.027203617617487907, |
| 0.022029871121048927, |
| 0.020715953782200813 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 94.2767562866211, |
| "energy_top5_pct": 96.86882019042969, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.55271096850716 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.self_attn.q_norm.weight", |
| "layer": 34, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.self_attn.q_proj.weight", |
| "layer": 34, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.34.self_attn.v_proj.weight", |
| "layer": 34, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.input_layernorm.weight", |
| "layer": 35, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.layer_scalar", |
| "layer": 35, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.mlp.down_proj.weight", |
| "layer": 35, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.mlp.gate_proj.weight", |
| "layer": 35, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.mlp.up_proj.weight", |
| "layer": 35, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.per_layer_input_gate.weight", |
| "layer": 35, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.per_layer_projection.weight", |
| "layer": 35, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.post_attention_layernorm.weight", |
| "layer": 35, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.post_feedforward_layernorm.weight", |
| "layer": 35, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.post_per_layer_input_norm.weight", |
| "layer": 35, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.pre_feedforward_layernorm.weight", |
| "layer": 35, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.self_attn.k_norm.weight", |
| "layer": 35, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.self_attn.k_proj.weight", |
| "layer": 35, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.self_attn.o_proj.weight", |
| "layer": 35, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 1.3419512510299683, |
| "mean_abs": 0.00019251968478783965, |
| "max_abs": 0.017578125, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 13.131940841674805, |
| "top_singular_values": [ |
| 1.2978599071502686, |
| 0.09883230179548264, |
| 0.08450636267662048, |
| 0.07443507760763168, |
| 0.052932433784008026, |
| 0.048107896000146866, |
| 0.03976667672395706, |
| 0.036174479871988297, |
| 0.028611760586500168, |
| 0.025871189311146736, |
| 0.024239545688033104, |
| 0.01935676857829094, |
| 0.018789755180478096, |
| 0.015938296914100647, |
| 0.015235750004649162, |
| 0.01423825602978468, |
| 0.013470897451043129, |
| 0.013089030981063843, |
| 0.01297325175255537, |
| 0.01216877344995737 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 93.53672790527344, |
| "energy_top5_pct": 94.93894958496094, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 95.45900635366202 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.self_attn.q_norm.weight", |
| "layer": 35, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.self_attn.q_proj.weight", |
| "layer": 35, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.35.self_attn.v_proj.weight", |
| "layer": 35, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.input_layernorm.weight", |
| "layer": 36, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.layer_scalar", |
| "layer": 36, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.mlp.down_proj.weight", |
| "layer": 36, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.mlp.gate_proj.weight", |
| "layer": 36, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.mlp.up_proj.weight", |
| "layer": 36, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.per_layer_input_gate.weight", |
| "layer": 36, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.per_layer_projection.weight", |
| "layer": 36, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.post_attention_layernorm.weight", |
| "layer": 36, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.post_feedforward_layernorm.weight", |
| "layer": 36, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.post_per_layer_input_norm.weight", |
| "layer": 36, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.pre_feedforward_layernorm.weight", |
| "layer": 36, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.self_attn.k_norm.weight", |
| "layer": 36, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.self_attn.k_proj.weight", |
| "layer": 36, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.self_attn.o_proj.weight", |
| "layer": 36, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.100762128829956, |
| "mean_abs": 0.0004277386178728193, |
| "max_abs": 0.06298828125, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 13.741976737976074, |
| "top_singular_values": [ |
| 2.053994655609131, |
| 0.14946864545345306, |
| 0.09495546668767929, |
| 0.08382919430732727, |
| 0.07874999195337296, |
| 0.06609689444303513, |
| 0.051928747445344925, |
| 0.04338708147406578, |
| 0.040210574865341187, |
| 0.03769753873348236, |
| 0.03482869267463684, |
| 0.033306658267974854, |
| 0.0314226858317852, |
| 0.02801136113703251, |
| 0.02653987891972065, |
| 0.021592242643237114, |
| 0.019437775015830994, |
| 0.01901092566549778, |
| 0.01829417049884796, |
| 0.016879461705684662 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 95.59713745117188, |
| "energy_top5_pct": 96.60743713378906, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.02909982607481 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.self_attn.q_norm.weight", |
| "layer": 36, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.self_attn.q_proj.weight", |
| "layer": 36, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.36.self_attn.v_proj.weight", |
| "layer": 36, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.input_layernorm.weight", |
| "layer": 37, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.layer_scalar", |
| "layer": 37, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.mlp.down_proj.weight", |
| "layer": 37, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.mlp.gate_proj.weight", |
| "layer": 37, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.mlp.up_proj.weight", |
| "layer": 37, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.per_layer_input_gate.weight", |
| "layer": 37, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.per_layer_projection.weight", |
| "layer": 37, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.post_attention_layernorm.weight", |
| "layer": 37, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.post_feedforward_layernorm.weight", |
| "layer": 37, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.post_per_layer_input_norm.weight", |
| "layer": 37, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.pre_feedforward_layernorm.weight", |
| "layer": 37, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.self_attn.k_norm.weight", |
| "layer": 37, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.self_attn.k_proj.weight", |
| "layer": 37, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.self_attn.o_proj.weight", |
| "layer": 37, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.1370456218719482, |
| "mean_abs": 0.0004189459141343832, |
| "max_abs": 0.03662109375, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 12.21744155883789, |
| "top_singular_values": [ |
| 2.099623918533325, |
| 0.171854630112648, |
| 0.0969209298491478, |
| 0.08609890192747116, |
| 0.07555954158306122, |
| 0.052164096385240555, |
| 0.04477650672197342, |
| 0.04084254428744316, |
| 0.032484009861946106, |
| 0.031264085322618484, |
| 0.029215911403298378, |
| 0.027071068063378334, |
| 0.025674929842352867, |
| 0.023261163383722305, |
| 0.022911367937922478, |
| 0.021498944610357285, |
| 0.019664935767650604, |
| 0.01927908882498741, |
| 0.015559712424874306, |
| 0.01459866389632225 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 96.52847290039062, |
| "energy_top5_pct": 97.66817474365234, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.96190793687109 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.self_attn.q_norm.weight", |
| "layer": 37, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.self_attn.q_proj.weight", |
| "layer": 37, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.37.self_attn.v_proj.weight", |
| "layer": 37, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.input_layernorm.weight", |
| "layer": 38, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.layer_scalar", |
| "layer": 38, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.mlp.down_proj.weight", |
| "layer": 38, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.mlp.gate_proj.weight", |
| "layer": 38, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.mlp.up_proj.weight", |
| "layer": 38, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.per_layer_input_gate.weight", |
| "layer": 38, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.per_layer_projection.weight", |
| "layer": 38, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.post_attention_layernorm.weight", |
| "layer": 38, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.post_feedforward_layernorm.weight", |
| "layer": 38, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.post_per_layer_input_norm.weight", |
| "layer": 38, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.pre_feedforward_layernorm.weight", |
| "layer": 38, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.self_attn.k_norm.weight", |
| "layer": 38, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.self_attn.k_proj.weight", |
| "layer": 38, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.self_attn.o_proj.weight", |
| "layer": 38, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.321763515472412, |
| "mean_abs": 0.0005104857846163213, |
| "max_abs": 0.0428314208984375, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 6.884712219238281, |
| "top_singular_values": [ |
| 2.2371041774749756, |
| 0.32493793964385986, |
| 0.20472566783428192, |
| 0.17324428260326385, |
| 0.1635258048772812, |
| 0.09762100130319595, |
| 0.08174652606248856, |
| 0.07870378345251083, |
| 0.07123638689517975, |
| 0.05593923106789589, |
| 0.04315295070409775, |
| 0.04080915451049805, |
| 0.036186084151268005, |
| 0.034083787351846695, |
| 0.033238694071769714, |
| 0.03234052285552025, |
| 0.030876636505126953, |
| 0.030038630589842796, |
| 0.028882503509521484, |
| 0.027293715626001358 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 92.84028625488281, |
| "energy_top5_pct": 96.62931823730469, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.41206546832915 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.self_attn.q_norm.weight", |
| "layer": 38, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.self_attn.q_proj.weight", |
| "layer": 38, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.38.self_attn.v_proj.weight", |
| "layer": 38, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.input_layernorm.weight", |
| "layer": 39, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.layer_scalar", |
| "layer": 39, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.mlp.down_proj.weight", |
| "layer": 39, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.mlp.gate_proj.weight", |
| "layer": 39, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.mlp.up_proj.weight", |
| "layer": 39, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.per_layer_input_gate.weight", |
| "layer": 39, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.per_layer_projection.weight", |
| "layer": 39, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.post_attention_layernorm.weight", |
| "layer": 39, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.post_feedforward_layernorm.weight", |
| "layer": 39, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.post_per_layer_input_norm.weight", |
| "layer": 39, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.pre_feedforward_layernorm.weight", |
| "layer": 39, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.self_attn.k_norm.weight", |
| "layer": 39, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.self_attn.k_proj.weight", |
| "layer": 39, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.self_attn.o_proj.weight", |
| "layer": 39, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 2.495680332183838, |
| "mean_abs": 0.0005324308876879513, |
| "max_abs": 0.060546875, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 10.821382522583008, |
| "top_singular_values": [ |
| 2.4362971782684326, |
| 0.22513732314109802, |
| 0.1982962042093277, |
| 0.13968002796173096, |
| 0.09458732604980469, |
| 0.08306797593832016, |
| 0.06880109757184982, |
| 0.06208239123225212, |
| 0.058144181966781616, |
| 0.057508960366249084, |
| 0.051184527575969696, |
| 0.04928537458181381, |
| 0.04783058911561966, |
| 0.04663194343447685, |
| 0.04418926313519478, |
| 0.04212530329823494, |
| 0.03786343336105347, |
| 0.03633158653974533, |
| 0.03390389680862427, |
| 0.029721839353442192 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 95.29774475097656, |
| "energy_top5_pct": 97.19975280761719, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 97.84521255772331 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.self_attn.q_norm.weight", |
| "layer": 39, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.self_attn.q_proj.weight", |
| "layer": 39, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.39.self_attn.v_proj.weight", |
| "layer": 39, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.input_layernorm.weight", |
| "layer": 4, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.layer_scalar", |
| "layer": 4, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.mlp.down_proj.weight", |
| "layer": 4, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.mlp.gate_proj.weight", |
| "layer": 4, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.mlp.up_proj.weight", |
| "layer": 4, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.per_layer_input_gate.weight", |
| "layer": 4, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.per_layer_projection.weight", |
| "layer": 4, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.post_attention_layernorm.weight", |
| "layer": 4, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.post_feedforward_layernorm.weight", |
| "layer": 4, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.post_per_layer_input_norm.weight", |
| "layer": 4, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.pre_feedforward_layernorm.weight", |
| "layer": 4, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.self_attn.k_norm.weight", |
| "layer": 4, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.self_attn.k_proj.weight", |
| "layer": 4, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.self_attn.o_proj.weight", |
| "layer": 4, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.self_attn.q_norm.weight", |
| "layer": 4, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.self_attn.q_proj.weight", |
| "layer": 4, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.4.self_attn.v_proj.weight", |
| "layer": 4, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.input_layernorm.weight", |
| "layer": 40, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.layer_scalar", |
| "layer": 40, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.mlp.down_proj.weight", |
| "layer": 40, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.mlp.gate_proj.weight", |
| "layer": 40, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.mlp.up_proj.weight", |
| "layer": 40, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.per_layer_input_gate.weight", |
| "layer": 40, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.per_layer_projection.weight", |
| "layer": 40, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.post_attention_layernorm.weight", |
| "layer": 40, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.post_feedforward_layernorm.weight", |
| "layer": 40, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.post_per_layer_input_norm.weight", |
| "layer": 40, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.pre_feedforward_layernorm.weight", |
| "layer": 40, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.self_attn.k_norm.weight", |
| "layer": 40, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.self_attn.k_proj.weight", |
| "layer": 40, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.self_attn.o_proj.weight", |
| "layer": 40, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 1.8438897132873535, |
| "mean_abs": 0.00042951255454681814, |
| "max_abs": 0.0634765625, |
| "effective_rank_90pct_energy": 1, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 6.422464847564697, |
| "top_singular_values": [ |
| 1.7965998649597168, |
| 0.27973681688308716, |
| 0.11466702073812485, |
| 0.0813412293791771, |
| 0.05839383974671364, |
| 0.04768190160393715, |
| 0.04147094860672951, |
| 0.03761795163154602, |
| 0.030466139316558838, |
| 0.030015671625733376, |
| 0.027458563446998596, |
| 0.0267021544277668, |
| 0.023045770823955536, |
| 0.0219968743622303, |
| 0.020260164514183998, |
| 0.016066541895270348, |
| 0.014314182102680206, |
| 0.012748711742460728, |
| 0.012226280756294727, |
| 0.011902361176908016 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 94.93641662597656, |
| "energy_top5_pct": 97.91963958740234, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 98.24455089925942 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.self_attn.q_norm.weight", |
| "layer": 40, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.self_attn.q_proj.weight", |
| "layer": 40, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.40.self_attn.v_proj.weight", |
| "layer": 40, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.input_layernorm.weight", |
| "layer": 41, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.layer_scalar", |
| "layer": 41, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.mlp.down_proj.weight", |
| "layer": 41, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.mlp.gate_proj.weight", |
| "layer": 41, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.mlp.up_proj.weight", |
| "layer": 41, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.per_layer_input_gate.weight", |
| "layer": 41, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.per_layer_projection.weight", |
| "layer": 41, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.post_attention_layernorm.weight", |
| "layer": 41, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.post_feedforward_layernorm.weight", |
| "layer": 41, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.post_per_layer_input_norm.weight", |
| "layer": 41, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.pre_feedforward_layernorm.weight", |
| "layer": 41, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.self_attn.k_norm.weight", |
| "layer": 41, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.self_attn.k_proj.weight", |
| "layer": 41, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.self_attn.o_proj.weight", |
| "layer": 41, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.10944202542304993, |
| "mean_abs": 8.73920271260431e-06, |
| "max_abs": 0.0009765625, |
| "effective_rank_90pct_energy": 20, |
| "effective_rank_99pct_energy": 20, |
| "sv_ratio_top2": 1.0800732374191284, |
| "top_singular_values": [ |
| 0.004510390106588602, |
| 0.004176003858447075, |
| 0.00417224271222949, |
| 0.004127850290387869, |
| 0.004115631338208914, |
| 0.004105337429791689, |
| 0.004098126199096441, |
| 0.004073730204254389, |
| 0.004047178663313389, |
| 0.004023457877337933, |
| 0.00399002805352211, |
| 0.003983719274401665, |
| 0.003973301500082016, |
| 0.003959247842431068, |
| 0.003938717767596245, |
| 0.0039141494780778885, |
| 0.0038852375000715256, |
| 0.0038507948629558086, |
| 0.0038233476225286722, |
| 0.0038147554732859135 |
| ], |
| "total_rank": 20, |
| "energy_top1_pct": 0.16984781622886658, |
| "energy_top5_pct": 0.7444573640823364, |
| "top_k_computed": 20, |
| "full_svd": false, |
| "lowrank_captured_energy_pct": 2.7147491501036205 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.self_attn.q_norm.weight", |
| "layer": 41, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.self_attn.q_proj.weight", |
| "layer": 41, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.41.self_attn.v_proj.weight", |
| "layer": 41, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.input_layernorm.weight", |
| "layer": 5, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.layer_scalar", |
| "layer": 5, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.mlp.down_proj.weight", |
| "layer": 5, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.mlp.gate_proj.weight", |
| "layer": 5, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.mlp.up_proj.weight", |
| "layer": 5, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.per_layer_input_gate.weight", |
| "layer": 5, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.per_layer_projection.weight", |
| "layer": 5, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.post_attention_layernorm.weight", |
| "layer": 5, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.post_feedforward_layernorm.weight", |
| "layer": 5, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.post_per_layer_input_norm.weight", |
| "layer": 5, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.pre_feedforward_layernorm.weight", |
| "layer": 5, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.self_attn.k_norm.weight", |
| "layer": 5, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.self_attn.k_proj.weight", |
| "layer": 5, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.self_attn.o_proj.weight", |
| "layer": 5, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 4096 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.self_attn.q_norm.weight", |
| "layer": 5, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 512 |
| ], |
| "numel": 512, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.self_attn.q_proj.weight", |
| "layer": 5, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 4096, |
| 2560 |
| ], |
| "numel": 10485760, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.5.self_attn.v_proj.weight", |
| "layer": 5, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 1024, |
| 2560 |
| ], |
| "numel": 2621440, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.input_layernorm.weight", |
| "layer": 6, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.layer_scalar", |
| "layer": 6, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.mlp.down_proj.weight", |
| "layer": 6, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.mlp.gate_proj.weight", |
| "layer": 6, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.mlp.up_proj.weight", |
| "layer": 6, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.per_layer_input_gate.weight", |
| "layer": 6, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.per_layer_projection.weight", |
| "layer": 6, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.post_attention_layernorm.weight", |
| "layer": 6, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.post_feedforward_layernorm.weight", |
| "layer": 6, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.post_per_layer_input_norm.weight", |
| "layer": 6, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.pre_feedforward_layernorm.weight", |
| "layer": 6, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.self_attn.k_norm.weight", |
| "layer": 6, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.self_attn.k_proj.weight", |
| "layer": 6, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.self_attn.o_proj.weight", |
| "layer": 6, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.self_attn.q_norm.weight", |
| "layer": 6, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.self_attn.q_proj.weight", |
| "layer": 6, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.6.self_attn.v_proj.weight", |
| "layer": 6, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.input_layernorm.weight", |
| "layer": 7, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.layer_scalar", |
| "layer": 7, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.mlp.down_proj.weight", |
| "layer": 7, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.mlp.gate_proj.weight", |
| "layer": 7, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.mlp.up_proj.weight", |
| "layer": 7, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.per_layer_input_gate.weight", |
| "layer": 7, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.per_layer_projection.weight", |
| "layer": 7, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.post_attention_layernorm.weight", |
| "layer": 7, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.post_feedforward_layernorm.weight", |
| "layer": 7, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.post_per_layer_input_norm.weight", |
| "layer": 7, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.pre_feedforward_layernorm.weight", |
| "layer": 7, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.self_attn.k_norm.weight", |
| "layer": 7, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.self_attn.k_proj.weight", |
| "layer": 7, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.self_attn.o_proj.weight", |
| "layer": 7, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.self_attn.q_norm.weight", |
| "layer": 7, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.self_attn.q_proj.weight", |
| "layer": 7, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.7.self_attn.v_proj.weight", |
| "layer": 7, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.input_layernorm.weight", |
| "layer": 8, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.layer_scalar", |
| "layer": 8, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.mlp.down_proj.weight", |
| "layer": 8, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.mlp.gate_proj.weight", |
| "layer": 8, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.mlp.up_proj.weight", |
| "layer": 8, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.per_layer_input_gate.weight", |
| "layer": 8, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.per_layer_projection.weight", |
| "layer": 8, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.post_attention_layernorm.weight", |
| "layer": 8, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.post_feedforward_layernorm.weight", |
| "layer": 8, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.post_per_layer_input_norm.weight", |
| "layer": 8, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.pre_feedforward_layernorm.weight", |
| "layer": 8, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.self_attn.k_norm.weight", |
| "layer": 8, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.self_attn.k_proj.weight", |
| "layer": 8, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.self_attn.o_proj.weight", |
| "layer": 8, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.self_attn.q_norm.weight", |
| "layer": 8, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.self_attn.q_proj.weight", |
| "layer": 8, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.8.self_attn.v_proj.weight", |
| "layer": 8, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.input_layernorm.weight", |
| "layer": 9, |
| "tensor_type": "input_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.layer_scalar", |
| "layer": 9, |
| "tensor_type": "layer_scalar", |
| "category": "scalar", |
| "shape": [ |
| 1 |
| ], |
| "numel": 1, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.mlp.down_proj.weight", |
| "layer": 9, |
| "tensor_type": "mlp.down_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 2560, |
| 10240 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.mlp.gate_proj.weight", |
| "layer": 9, |
| "tensor_type": "mlp.gate_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.mlp.up_proj.weight", |
| "layer": 9, |
| "tensor_type": "mlp.up_proj.weight", |
| "category": "mlp", |
| "shape": [ |
| 10240, |
| 2560 |
| ], |
| "numel": 26214400, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.per_layer_input_gate.weight", |
| "layer": 9, |
| "tensor_type": "per_layer_input_gate.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256, |
| 2560 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.per_layer_projection.weight", |
| "layer": 9, |
| "tensor_type": "per_layer_projection.weight", |
| "category": "per_layer", |
| "shape": [ |
| 2560, |
| 256 |
| ], |
| "numel": 655360, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.post_attention_layernorm.weight", |
| "layer": 9, |
| "tensor_type": "post_attention_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.post_feedforward_layernorm.weight", |
| "layer": 9, |
| "tensor_type": "post_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.post_per_layer_input_norm.weight", |
| "layer": 9, |
| "tensor_type": "post_per_layer_input_norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.pre_feedforward_layernorm.weight", |
| "layer": 9, |
| "tensor_type": "pre_feedforward_layernorm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.self_attn.k_norm.weight", |
| "layer": 9, |
| "tensor_type": "self_attn.k_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.self_attn.k_proj.weight", |
| "layer": 9, |
| "tensor_type": "self_attn.k_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.self_attn.o_proj.weight", |
| "layer": 9, |
| "tensor_type": "self_attn.o_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2560, |
| 2048 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.self_attn.q_norm.weight", |
| "layer": 9, |
| "tensor_type": "self_attn.q_norm.weight", |
| "category": "attention", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.self_attn.q_proj.weight", |
| "layer": 9, |
| "tensor_type": "self_attn.q_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 2048, |
| 2560 |
| ], |
| "numel": 5242880, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.layers.9.self_attn.v_proj.weight", |
| "layer": 9, |
| "tensor_type": "self_attn.v_proj.weight", |
| "category": "attention", |
| "shape": [ |
| 512, |
| 2560 |
| ], |
| "numel": 1310720, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.norm.weight", |
| "layer": null, |
| "tensor_type": "norm.weight", |
| "category": "norm", |
| "shape": [ |
| 2560 |
| ], |
| "numel": 2560, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.per_layer_model_projection.weight", |
| "layer": null, |
| "tensor_type": "per_layer_model_projection.weight", |
| "category": "other", |
| "shape": [ |
| 10752, |
| 2560 |
| ], |
| "numel": 27525120, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| }, |
| { |
| "key": "model.language_model.per_layer_projection_norm.weight", |
| "layer": null, |
| "tensor_type": "per_layer_projection_norm.weight", |
| "category": "per_layer", |
| "shape": [ |
| 256 |
| ], |
| "numel": 256, |
| "Coder3101 Heretic_vs_base": { |
| "frobenius_norm": 0.0, |
| "mean_abs": 0.0, |
| "max_abs": 0.0 |
| } |
| } |
| ] |
| } |
