| { |
| "format": "jev_fp4_w4a16_v1", |
| "weight_format": "E2M1 packed uint8, low nibble first", |
| "block_size": 16, |
| "block_scale_dtype": "float8_e4m3fn", |
| "tensor_scale_dtype": "float32", |
| "activation_quantization": "none; BF16 activations", |
| "scope": "all", |
| "modules": { |
| "language_model.layers.0.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09486016631126404 |
| }, |
| "language_model.layers.0.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09497135132551193 |
| }, |
| "language_model.layers.0.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09450867772102356 |
| }, |
| "language_model.layers.0.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0948721319437027 |
| }, |
| "language_model.layers.0.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499333053827286 |
| }, |
| "language_model.layers.0.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09505700320005417 |
| }, |
| "language_model.layers.1.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09484443813562393 |
| }, |
| "language_model.layers.1.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09514866769313812 |
| }, |
| "language_model.layers.1.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09484771639108658 |
| }, |
| "language_model.layers.1.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09486880898475647 |
| }, |
| "language_model.layers.1.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09489931166172028 |
| }, |
| "language_model.layers.1.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09498240798711777 |
| }, |
| "language_model.layers.2.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09503107517957687 |
| }, |
| "language_model.layers.2.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0950225368142128 |
| }, |
| "language_model.layers.2.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09481486678123474 |
| }, |
| "language_model.layers.2.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0949583575129509 |
| }, |
| "language_model.layers.2.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09512658417224884 |
| }, |
| "language_model.layers.2.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504052996635437 |
| }, |
| "language_model.layers.3.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09508463740348816 |
| }, |
| "language_model.layers.3.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0951017364859581 |
| }, |
| "language_model.layers.3.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0951053649187088 |
| }, |
| "language_model.layers.3.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09465542435646057 |
| }, |
| "language_model.layers.3.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09480992704629898 |
| }, |
| "language_model.layers.3.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09493377804756165 |
| }, |
| "language_model.layers.3.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09477821737527847 |
| }, |
| "language_model.layers.4.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09525646269321442 |
| }, |
| "language_model.layers.4.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09491131454706192 |
| }, |
| "language_model.layers.4.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09473739564418793 |
| }, |
| "language_model.layers.4.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09508217871189117 |
| }, |
| "language_model.layers.4.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09503670036792755 |
| }, |
| "language_model.layers.4.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09508693963289261 |
| }, |
| "language_model.layers.5.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09494045376777649 |
| }, |
| "language_model.layers.5.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09495068341493607 |
| }, |
| "language_model.layers.5.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09470401704311371 |
| }, |
| "language_model.layers.5.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09510992467403412 |
| }, |
| "language_model.layers.5.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09505727142095566 |
| }, |
| "language_model.layers.5.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504293650388718 |
| }, |
| "language_model.layers.6.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0949840098619461 |
| }, |
| "language_model.layers.6.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09488002955913544 |
| }, |
| "language_model.layers.6.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09481548517942429 |
| }, |
| "language_model.layers.6.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09487542510032654 |
| }, |
| "language_model.layers.6.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09487628191709518 |
| }, |
| "language_model.layers.6.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09496339410543442 |
| }, |
| "language_model.layers.7.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0950276255607605 |
| }, |
| "language_model.layers.7.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09497901052236557 |
| }, |
| "language_model.layers.7.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09511414170265198 |
| }, |
| "language_model.layers.7.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09430784732103348 |
| }, |
| "language_model.layers.7.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09503162652254105 |
| }, |
| "language_model.layers.7.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09478483349084854 |
| }, |
| "language_model.layers.7.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09448397159576416 |
| }, |
| "language_model.layers.8.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0950857624411583 |
| }, |
| "language_model.layers.8.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09473881870508194 |
| }, |
| "language_model.layers.8.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09493368119001389 |
| }, |
| "language_model.layers.8.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09497066587209702 |
| }, |
| "language_model.layers.8.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09510495513677597 |
| }, |
| "language_model.layers.8.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09500662237405777 |
| }, |
| "language_model.layers.9.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09504227340221405 |
| }, |
| "language_model.layers.9.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09509280323982239 |
| }, |
| "language_model.layers.9.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09492899477481842 |
| }, |
| "language_model.layers.9.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09499592334032059 |
| }, |
| "language_model.layers.10.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09482154995203018 |
| }, |
| "language_model.layers.10.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09499532729387283 |
| }, |
| "language_model.layers.10.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09488175809383392 |
| }, |
| "language_model.layers.10.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09498362243175507 |
| }, |
| "language_model.layers.10.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09510846436023712 |
| }, |
| "language_model.layers.10.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504243731498718 |
| }, |
| "language_model.layers.11.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09496378153562546 |
| }, |
| "language_model.layers.11.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09514947980642319 |
| }, |
| "language_model.layers.11.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09500227123498917 |
| }, |
| "language_model.layers.11.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09418924897909164 |
| }, |
| "language_model.layers.11.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0949619859457016 |
| }, |
| "language_model.layers.11.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09488285332918167 |
| }, |
| "language_model.layers.11.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09459587931632996 |
| }, |
| "language_model.layers.12.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09506550431251526 |
| }, |
| "language_model.layers.12.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0949074998497963 |
| }, |
| "language_model.layers.12.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09501614421606064 |
| }, |
| "language_model.layers.12.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09496786445379257 |
| }, |
| "language_model.layers.12.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09498301893472672 |
| }, |
| "language_model.layers.12.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09506572037935257 |
| }, |
| "language_model.layers.13.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09531169384717941 |
| }, |
| "language_model.layers.13.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09491849690675735 |
| }, |
| "language_model.layers.13.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09507833421230316 |
| }, |
| "language_model.layers.13.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09501072019338608 |
| }, |
| "language_model.layers.13.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09508799761533737 |
| }, |
| "language_model.layers.13.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09514525532722473 |
| }, |
| "language_model.layers.14.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09476681053638458 |
| }, |
| "language_model.layers.14.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09494723379611969 |
| }, |
| "language_model.layers.14.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09504920989274979 |
| }, |
| "language_model.layers.14.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09514512121677399 |
| }, |
| "language_model.layers.14.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09514012187719345 |
| }, |
| "language_model.layers.14.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0950288474559784 |
| }, |
| "language_model.layers.15.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09509720653295517 |
| }, |
| "language_model.layers.15.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09515085071325302 |
| }, |
| "language_model.layers.15.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09498771280050278 |
| }, |
| "language_model.layers.15.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09469085931777954 |
| }, |
| "language_model.layers.15.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09490107744932175 |
| }, |
| "language_model.layers.15.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09494346380233765 |
| }, |
| "language_model.layers.15.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.094981849193573 |
| }, |
| "language_model.layers.9.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09502971917390823 |
| }, |
| "language_model.layers.9.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0950048565864563 |
| }, |
| "language_model.layers.16.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09492098540067673 |
| }, |
| "language_model.layers.16.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09501805156469345 |
| }, |
| "language_model.layers.16.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09504058212041855 |
| }, |
| "language_model.layers.16.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0948575884103775 |
| }, |
| "language_model.layers.16.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504494071006775 |
| }, |
| "language_model.layers.16.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09507948905229568 |
| }, |
| "language_model.layers.17.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09499464929103851 |
| }, |
| "language_model.layers.17.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09500721096992493 |
| }, |
| "language_model.layers.17.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09492168575525284 |
| }, |
| "language_model.layers.17.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0948880985379219 |
| }, |
| "language_model.layers.17.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09502647817134857 |
| }, |
| "language_model.layers.17.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504751861095428 |
| }, |
| "language_model.layers.18.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09492260217666626 |
| }, |
| "language_model.layers.18.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09499627351760864 |
| }, |
| "language_model.layers.18.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09481608867645264 |
| }, |
| "language_model.layers.18.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09485505521297455 |
| }, |
| "language_model.layers.18.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09494347870349884 |
| }, |
| "language_model.layers.18.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09514016658067703 |
| }, |
| "language_model.layers.19.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09493260085582733 |
| }, |
| "language_model.layers.19.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0949341356754303 |
| }, |
| "language_model.layers.19.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09500252455472946 |
| }, |
| "language_model.layers.19.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09432574361562729 |
| }, |
| "language_model.layers.19.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09484393149614334 |
| }, |
| "language_model.layers.19.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09493577480316162 |
| }, |
| "language_model.layers.19.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09439519792795181 |
| }, |
| "language_model.layers.20.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09497325122356415 |
| }, |
| "language_model.layers.20.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09502489864826202 |
| }, |
| "language_model.layers.20.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0948246568441391 |
| }, |
| "language_model.layers.20.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09502345323562622 |
| }, |
| "language_model.layers.20.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09514394402503967 |
| }, |
| "language_model.layers.20.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09500880539417267 |
| }, |
| "language_model.layers.21.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0948365181684494 |
| }, |
| "language_model.layers.21.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09501895308494568 |
| }, |
| "language_model.layers.21.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0947766900062561 |
| }, |
| "language_model.layers.21.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0948769599199295 |
| }, |
| "language_model.layers.21.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09502305835485458 |
| }, |
| "language_model.layers.21.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499023109674454 |
| }, |
| "language_model.layers.22.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0949094220995903 |
| }, |
| "language_model.layers.22.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09501488506793976 |
| }, |
| "language_model.layers.22.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09486842900514603 |
| }, |
| "language_model.layers.22.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09473083913326263 |
| }, |
| "language_model.layers.22.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499149769544601 |
| }, |
| "language_model.layers.22.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09487354755401611 |
| }, |
| "language_model.layers.23.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09514690190553665 |
| }, |
| "language_model.layers.23.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499688446521759 |
| }, |
| "language_model.layers.23.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09503503143787384 |
| }, |
| "language_model.layers.23.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09404724091291428 |
| }, |
| "language_model.layers.23.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09494449943304062 |
| }, |
| "language_model.layers.23.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09494021534919739 |
| }, |
| "language_model.layers.23.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09453386813402176 |
| }, |
| "language_model.layers.24.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09500335901975632 |
| }, |
| "language_model.layers.24.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09493091702461243 |
| }, |
| "language_model.layers.24.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09486418962478638 |
| }, |
| "language_model.layers.24.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09495484828948975 |
| }, |
| "language_model.layers.24.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09501788765192032 |
| }, |
| "language_model.layers.24.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09502902626991272 |
| }, |
| "language_model.layers.25.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09482426941394806 |
| }, |
| "language_model.layers.25.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09474703669548035 |
| }, |
| "language_model.layers.25.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09497242420911789 |
| }, |
| "language_model.layers.25.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09508100897073746 |
| }, |
| "language_model.layers.25.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09501214325428009 |
| }, |
| "language_model.layers.25.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0950527936220169 |
| }, |
| "language_model.layers.26.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09467176347970963 |
| }, |
| "language_model.layers.26.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09480072557926178 |
| }, |
| "language_model.layers.26.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09490510821342468 |
| }, |
| "language_model.layers.26.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09496297687292099 |
| }, |
| "language_model.layers.26.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09498626738786697 |
| }, |
| "language_model.layers.26.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09480836987495422 |
| }, |
| "language_model.layers.27.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09484710544347763 |
| }, |
| "language_model.layers.27.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09493212401866913 |
| }, |
| "language_model.layers.27.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0948168933391571 |
| }, |
| "language_model.layers.27.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09384384751319885 |
| }, |
| "language_model.layers.27.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09486248344182968 |
| }, |
| "language_model.layers.27.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09475899487733841 |
| }, |
| "language_model.layers.27.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09454510360956192 |
| }, |
| "language_model.layers.28.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09461823105812073 |
| }, |
| "language_model.layers.28.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09468646347522736 |
| }, |
| "language_model.layers.28.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0948798656463623 |
| }, |
| "language_model.layers.28.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09483753889799118 |
| }, |
| "language_model.layers.28.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09500079602003098 |
| }, |
| "language_model.layers.28.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09481845051050186 |
| }, |
| "language_model.layers.29.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09494879841804504 |
| }, |
| "language_model.layers.29.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09500574320554733 |
| }, |
| "language_model.layers.29.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09493628889322281 |
| }, |
| "language_model.layers.29.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09485795348882675 |
| }, |
| "language_model.layers.29.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09481706470251083 |
| }, |
| "language_model.layers.29.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09477069973945618 |
| }, |
| "language_model.layers.30.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09461774677038193 |
| }, |
| "language_model.layers.30.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09483305364847183 |
| }, |
| "language_model.layers.30.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09494119882583618 |
| }, |
| "language_model.layers.30.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.0949774906039238 |
| }, |
| "language_model.layers.30.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09488280862569809 |
| }, |
| "language_model.layers.30.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09494347125291824 |
| }, |
| "language_model.layers.31.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09492547065019608 |
| }, |
| "language_model.layers.31.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0947427749633789 |
| }, |
| "language_model.layers.31.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09495143592357635 |
| }, |
| "language_model.layers.31.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09425639361143112 |
| }, |
| "language_model.layers.31.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09483080357313156 |
| }, |
| "language_model.layers.31.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09492354840040207 |
| }, |
| "language_model.layers.31.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09457071125507355 |
| }, |
| "language_model.layers.32.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09463939070701599 |
| }, |
| "language_model.layers.32.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0946582704782486 |
| }, |
| "language_model.layers.32.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09487397968769073 |
| }, |
| "language_model.layers.32.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09469287097454071 |
| }, |
| "language_model.layers.32.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09478341042995453 |
| }, |
| "language_model.layers.32.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09476879239082336 |
| }, |
| "language_model.layers.33.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09440957754850388 |
| }, |
| "language_model.layers.33.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09464427828788757 |
| }, |
| "language_model.layers.33.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09479189664125443 |
| }, |
| "language_model.layers.33.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.094871886074543 |
| }, |
| "language_model.layers.33.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499730169773102 |
| }, |
| "language_model.layers.33.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09476088732481003 |
| }, |
| "language_model.layers.34.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09468875825405121 |
| }, |
| "language_model.layers.34.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09478949010372162 |
| }, |
| "language_model.layers.34.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09469298273324966 |
| }, |
| "language_model.layers.34.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09461571276187897 |
| }, |
| "language_model.layers.34.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09463000297546387 |
| }, |
| "language_model.layers.34.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09475961327552795 |
| }, |
| "language_model.layers.35.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09494943916797638 |
| }, |
| "language_model.layers.35.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09489712119102478 |
| }, |
| "language_model.layers.35.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0948406457901001 |
| }, |
| "language_model.layers.35.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09458363801240921 |
| }, |
| "language_model.layers.35.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09498253464698792 |
| }, |
| "language_model.layers.35.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09476710855960846 |
| }, |
| "language_model.layers.35.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09429006278514862 |
| }, |
| "language_model.layers.36.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09468530118465424 |
| }, |
| "language_model.layers.36.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09500762075185776 |
| }, |
| "language_model.layers.36.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09488498419523239 |
| }, |
| "language_model.layers.36.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09489385038614273 |
| }, |
| "language_model.layers.36.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09498020261526108 |
| }, |
| "language_model.layers.36.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09488225728273392 |
| }, |
| "language_model.layers.37.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09489655494689941 |
| }, |
| "language_model.layers.37.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09505017846822739 |
| }, |
| "language_model.layers.37.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09477003663778305 |
| }, |
| "language_model.layers.37.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09496506303548813 |
| }, |
| "language_model.layers.37.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0950976312160492 |
| }, |
| "language_model.layers.37.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0949040874838829 |
| }, |
| "language_model.layers.38.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09471458941698074 |
| }, |
| "language_model.layers.38.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09500065445899963 |
| }, |
| "language_model.layers.38.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09491356462240219 |
| }, |
| "language_model.layers.38.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09493554383516312 |
| }, |
| "language_model.layers.38.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504397213459015 |
| }, |
| "language_model.layers.38.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09488421678543091 |
| }, |
| "language_model.layers.39.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09519059211015701 |
| }, |
| "language_model.layers.39.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09494423866271973 |
| }, |
| "language_model.layers.39.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09490010142326355 |
| }, |
| "language_model.layers.39.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09392006695270538 |
| }, |
| "language_model.layers.39.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09475131332874298 |
| }, |
| "language_model.layers.39.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.0948835015296936 |
| }, |
| "language_model.layers.39.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09420917928218842 |
| }, |
| "language_model.layers.40.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09492066502571106 |
| }, |
| "language_model.layers.40.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0948696881532669 |
| }, |
| "language_model.layers.40.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09492892026901245 |
| }, |
| "language_model.layers.40.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09490188956260681 |
| }, |
| "language_model.layers.40.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0951344296336174 |
| }, |
| "language_model.layers.40.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09501751512289047 |
| }, |
| "language_model.layers.41.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0948137417435646 |
| }, |
| "language_model.layers.41.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0948779433965683 |
| }, |
| "language_model.layers.41.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0947781577706337 |
| }, |
| "language_model.layers.41.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09503041952848434 |
| }, |
| "language_model.layers.41.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0948844701051712 |
| }, |
| "language_model.layers.41.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09497609734535217 |
| }, |
| "language_model.layers.42.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09471409767866135 |
| }, |
| "language_model.layers.42.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0949917882680893 |
| }, |
| "language_model.layers.42.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09480810165405273 |
| }, |
| "language_model.layers.42.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09481658041477203 |
| }, |
| "language_model.layers.42.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09484227746725082 |
| }, |
| "language_model.layers.42.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09501878917217255 |
| }, |
| "language_model.layers.43.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09485714882612228 |
| }, |
| "language_model.layers.43.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09488418698310852 |
| }, |
| "language_model.layers.43.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09511309117078781 |
| }, |
| "language_model.layers.43.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09373204410076141 |
| }, |
| "language_model.layers.43.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0945858582854271 |
| }, |
| "language_model.layers.43.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.0946039929986 |
| }, |
| "language_model.layers.43.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09429279714822769 |
| }, |
| "language_model.layers.44.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09474681317806244 |
| }, |
| "language_model.layers.44.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0949125811457634 |
| }, |
| "language_model.layers.44.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09474358707666397 |
| }, |
| "language_model.layers.44.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09497571736574173 |
| }, |
| "language_model.layers.44.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09484536945819855 |
| }, |
| "language_model.layers.44.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09481244534254074 |
| }, |
| "language_model.layers.45.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09481263160705566 |
| }, |
| "language_model.layers.45.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09497564285993576 |
| }, |
| "language_model.layers.45.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09486696869134903 |
| }, |
| "language_model.layers.45.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09488007426261902 |
| }, |
| "language_model.layers.45.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09497823566198349 |
| }, |
| "language_model.layers.45.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09472479671239853 |
| }, |
| "language_model.layers.46.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09470413625240326 |
| }, |
| "language_model.layers.46.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0950494259595871 |
| }, |
| "language_model.layers.46.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09488178789615631 |
| }, |
| "language_model.layers.46.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09481244534254074 |
| }, |
| "language_model.layers.46.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09477102756500244 |
| }, |
| "language_model.layers.46.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09480199962854385 |
| }, |
| "language_model.layers.47.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09494847804307938 |
| }, |
| "language_model.layers.47.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09477753937244415 |
| }, |
| "language_model.layers.47.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0947762131690979 |
| }, |
| "language_model.layers.47.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09442693740129471 |
| }, |
| "language_model.layers.47.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09441886842250824 |
| }, |
| "language_model.layers.47.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09475977718830109 |
| }, |
| "language_model.layers.47.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09455729275941849 |
| }, |
| "language_model.layers.48.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09464775025844574 |
| }, |
| "language_model.layers.48.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09490729868412018 |
| }, |
| "language_model.layers.48.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09488801658153534 |
| }, |
| "language_model.layers.48.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09485373646020889 |
| }, |
| "language_model.layers.48.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09467625617980957 |
| }, |
| "language_model.layers.48.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09496848285198212 |
| }, |
| "language_model.layers.49.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09464167058467865 |
| }, |
| "language_model.layers.49.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09494692087173462 |
| }, |
| "language_model.layers.49.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09463748335838318 |
| }, |
| "language_model.layers.49.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09468481689691544 |
| }, |
| "language_model.layers.49.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09491779655218124 |
| }, |
| "language_model.layers.49.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09485753625631332 |
| }, |
| "language_model.layers.50.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0947871208190918 |
| }, |
| "language_model.layers.50.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09501579403877258 |
| }, |
| "language_model.layers.50.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09453065693378448 |
| }, |
| "language_model.layers.50.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09454906731843948 |
| }, |
| "language_model.layers.50.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09466993808746338 |
| }, |
| "language_model.layers.50.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504862129688263 |
| }, |
| "language_model.layers.51.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09472827613353729 |
| }, |
| "language_model.layers.51.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09471764415502548 |
| }, |
| "language_model.layers.51.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09480338543653488 |
| }, |
| "language_model.layers.51.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09441620111465454 |
| }, |
| "language_model.layers.51.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09461163729429245 |
| }, |
| "language_model.layers.51.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09478556364774704 |
| }, |
| "language_model.layers.51.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09379476308822632 |
| }, |
| "language_model.layers.52.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09476900100708008 |
| }, |
| "language_model.layers.52.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09498173743486404 |
| }, |
| "language_model.layers.52.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09475252777338028 |
| }, |
| "language_model.layers.52.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09490261226892471 |
| }, |
| "language_model.layers.52.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09491124004125595 |
| }, |
| "language_model.layers.52.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09485194087028503 |
| }, |
| "language_model.layers.53.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09477796405553818 |
| }, |
| "language_model.layers.53.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09515088051557541 |
| }, |
| "language_model.layers.53.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09477433562278748 |
| }, |
| "language_model.layers.53.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09496921300888062 |
| }, |
| "language_model.layers.53.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09482976049184799 |
| }, |
| "language_model.layers.53.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09494800120592117 |
| }, |
| "language_model.layers.54.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09501806646585464 |
| }, |
| "language_model.layers.54.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09488914906978607 |
| }, |
| "language_model.layers.54.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09487079828977585 |
| }, |
| "language_model.layers.54.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09482055902481079 |
| }, |
| "language_model.layers.54.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.094856858253479 |
| }, |
| "language_model.layers.54.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09493105113506317 |
| }, |
| "language_model.layers.55.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.094979427754879 |
| }, |
| "language_model.layers.55.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09496595710515976 |
| }, |
| "language_model.layers.55.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09491196274757385 |
| }, |
| "language_model.layers.55.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09335911273956299 |
| }, |
| "language_model.layers.55.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09476569294929504 |
| }, |
| "language_model.layers.55.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09488401561975479 |
| }, |
| "language_model.layers.55.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09413868933916092 |
| }, |
| "language_model.layers.56.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09488192945718765 |
| }, |
| "language_model.layers.56.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.0950426235795021 |
| }, |
| "language_model.layers.56.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09476073831319809 |
| }, |
| "language_model.layers.56.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09507889300584793 |
| }, |
| "language_model.layers.56.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.0951027199625969 |
| }, |
| "language_model.layers.56.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09496468305587769 |
| }, |
| "language_model.layers.57.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09480869770050049 |
| }, |
| "language_model.layers.57.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09499526023864746 |
| }, |
| "language_model.layers.57.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09472248703241348 |
| }, |
| "language_model.layers.57.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09520614147186279 |
| }, |
| "language_model.layers.57.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09498605132102966 |
| }, |
| "language_model.layers.57.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09504371881484985 |
| }, |
| "language_model.layers.58.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09481693804264069 |
| }, |
| "language_model.layers.58.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09519118815660477 |
| }, |
| "language_model.layers.58.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0948721244931221 |
| }, |
| "language_model.layers.58.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09501434862613678 |
| }, |
| "language_model.layers.58.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499430656433105 |
| }, |
| "language_model.layers.58.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09505595266819 |
| }, |
| "language_model.layers.59.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09494626522064209 |
| }, |
| "language_model.layers.59.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09507293254137039 |
| }, |
| "language_model.layers.59.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09511616080999374 |
| }, |
| "language_model.layers.59.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.0926285833120346 |
| }, |
| "language_model.layers.59.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0948423221707344 |
| }, |
| "language_model.layers.59.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09460699558258057 |
| }, |
| "language_model.layers.59.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09384855628013611 |
| }, |
| "language_model.layers.60.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.0947812870144844 |
| }, |
| "language_model.layers.60.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09487397968769073 |
| }, |
| "language_model.layers.60.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09484171867370605 |
| }, |
| "language_model.layers.60.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09491603821516037 |
| }, |
| "language_model.layers.60.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09490219503641129 |
| }, |
| "language_model.layers.60.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09491108357906342 |
| }, |
| "language_model.layers.61.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09496872127056122 |
| }, |
| "language_model.layers.61.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09485461562871933 |
| }, |
| "language_model.layers.61.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0948089063167572 |
| }, |
| "language_model.layers.61.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09506987780332565 |
| }, |
| "language_model.layers.61.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09477683156728745 |
| }, |
| "language_model.layers.61.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09496009349822998 |
| }, |
| "language_model.layers.62.linear_attn.in_proj_qkv": { |
| "shape": [ |
| 10240, |
| 5120 |
| ], |
| "relative_rmse": 0.09448860585689545 |
| }, |
| "language_model.layers.62.linear_attn.in_proj_z": { |
| "shape": [ |
| 6144, |
| 5120 |
| ], |
| "relative_rmse": 0.09479902684688568 |
| }, |
| "language_model.layers.62.linear_attn.out_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.0946703851222992 |
| }, |
| "language_model.layers.62.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09482859075069427 |
| }, |
| "language_model.layers.62.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09477575868368149 |
| }, |
| "language_model.layers.62.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09491778910160065 |
| }, |
| "language_model.layers.63.mlp.down_proj": { |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "relative_rmse": 0.09433726966381073 |
| }, |
| "language_model.layers.63.mlp.gate_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09479404985904694 |
| }, |
| "language_model.layers.63.mlp.up_proj": { |
| "shape": [ |
| 17408, |
| 5120 |
| ], |
| "relative_rmse": 0.09499823302030563 |
| }, |
| "language_model.layers.63.self_attn.k_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09264403581619263 |
| }, |
| "language_model.layers.63.self_attn.o_proj": { |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "relative_rmse": 0.09375306218862534 |
| }, |
| "language_model.layers.63.self_attn.q_proj": { |
| "shape": [ |
| 12288, |
| 5120 |
| ], |
| "relative_rmse": 0.09451363235712051 |
| }, |
| "language_model.layers.63.self_attn.v_proj": { |
| "shape": [ |
| 1024, |
| 5120 |
| ], |
| "relative_rmse": 0.09369884431362152 |
| } |
| }, |
| "preserved": "decision readout, embeddings, vision tower, norms, convolutions, small gate projections; attention also preserved in mlp scope", |
| "production_backend": "Triton per-matrix dequantization + BF16 GEMM; no native FP4 tensor-core claim", |
| "source_checkpoint": "/shared/yue/jev-best-tritask-20261006/runs/balanced/checkpoints/step-00800", |
| "source_DI": 62.43, |
| "quantized_DI": null, |
| "modelopt_version": "0.47.0", |
| "no_training": true, |
| "runtime_entrypoint": "jev_fp4.FP4DecisionModel" |
| } |
|
|
