| [ |
| { |
| "layer_name": "lm_head", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00546584720723331 |
| }, |
| "param_count": 311164928, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0001811789904119602, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00087689898039874 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0015896713763747055 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00044233594720329467 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0003866669740091311 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.000519902074302081 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00046726479376957286 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.mlp.gate", |
| "sensitivities": { |
| "4": 0.0015319683431395485, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0009970010384373988 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0006898588944750372 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.000378026442679887 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.47.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0009673482066621849 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.00043826462933793664, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005482422323742261 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.000585554851568304 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005205812140047783 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005259119376811819 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005636888490698766 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005691807382390834 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.mlp.gate", |
| "sensitivities": { |
| "4": 0.0016299924997535224, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0013121363129660797 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0004549955301627051 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.46.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001232918316721528 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0004910269999527372, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0006056272814021213 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.000662573301572896 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005977782335927865 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0007479680231578337 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0007716155766199032 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0007242341719878217 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.mlp.gate", |
| "sensitivities": { |
| "4": 0.002146355339694613, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0011942335452962045 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0005906730202696053 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.45.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0012439823755509376 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0007133473106174885, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0007204592196406642 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0008755783965170849 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0006210746008340114 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.000983773055243849 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0010371999160270207 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0009049700092873536 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.mlp.gate", |
| "sensitivities": { |
| "4": 0.0024624387491106368, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0015454338014630291 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0007653660904907156 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.44.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0014417152075717847 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0007744313740355816, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0009393947063169131 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0010768429419840686 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0008377297441863144 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0010964083982495747 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0010202171264002875 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0011413340374322918 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.mlp.gate", |
| "sensitivities": { |
| "4": 0.0033793977951669754, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002702764031710103 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0017084380015148781 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0011191138861856114 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.43.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0014598523788057112 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0009925571054433628, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0011605675669367581 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0018117218254095253 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001172308443832056 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0015623406288796104 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00130574569387439 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0013518400919565465 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.mlp.gate", |
| "sensitivities": { |
| "4": 0.0029391975913313217, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0017956645048495072 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0011235147382346138 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.42.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002219276189862285 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0011014955040688317, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001253751875386418 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0012832630854973104 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0012258604571494895 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0015417903002041082 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0016429382061081317 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0015326397333410569 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.mlp.gate", |
| "sensitivities": { |
| "4": 0.0030040774484708286, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0017300844774581492 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0013138200107884284 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.41.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0019365232534861814 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0012782128954616685, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0014563855996433024 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0014926813782949466 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0013535707779131674 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0017617459622366975 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0016695782178430818 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0016951369301144343 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.mlp.gate", |
| "sensitivities": { |
| "4": 0.0033831604887382127, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002003027924123065 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0014643894125280592 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.40.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002364053131411007 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.001446501639293274, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001575188147398876 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001775789626359862 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0015565581913203157 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0019679781325976364 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001903772181928313 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001958755795688679 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.mlp.gate", |
| "sensitivities": { |
| "4": 0.0034947048746592677, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0022330831889121328 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0020966503264692924 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0017853173170199927 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.39.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002157975022176591 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0016617042371459927, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.001788420772451597 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0018763207917800173 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0018028927515842952 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00211460222756917 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0020056302140195235 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002035529483691789 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.mlp.gate", |
| "sensitivities": { |
| "4": 0.003621493626269512, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0020221223191280537 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0017770422612860177 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.38.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0024485136527800933 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0017955446419364307, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00191835029302941 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002024596572785716 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0018729715350976524 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0022996737267628 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0021713513148521693 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0021041337956072916 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.mlp.gate", |
| "sensitivities": { |
| "4": 0.004091399455016169, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002346425324503798 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0018104871329948462 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.37.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002669086587654116 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0018940380444594969, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002158472236866752 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002250783230313876 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002060351244532891 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002607721893582493 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0025067820267092125 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0025180679634407475 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.mlp.gate", |
| "sensitivities": { |
| "4": 0.0041634838610965135, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002644306846680896 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002127183021608895 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.36.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003041435554526591 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0021315743118369332, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0023513569540227763 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00261818288709037 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0022630117625036896 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0029103495204860033 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0026857099728658795 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0027523967655724846 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.mlp.gate", |
| "sensitivities": { |
| "4": 0.004539914623213311, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003231826927124833 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0032008243239639946 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0025570961685540774 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.35.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002896230924913349 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.002297268777814073, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0026993594292434864 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0027452579136782638 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0024244497229422755 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0029562374596328786 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00277543455013074 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00286523503503607 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.mlp.gate", |
| "sensitivities": { |
| "4": 0.00521997369651217, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003193213837221265 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002630184574324327 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.34.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0036525255200103857 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.002746699678633983, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0027895902336846725 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0029053816663993834 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002501191748403168 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0031272211781470105 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0031219236722487644 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0032260748290961296 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.mlp.gate", |
| "sensitivities": { |
| "4": 0.005147328270444025, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0033409177121939138 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002774385551068311 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.33.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0038971262256382033 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0028969740136138475, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0031035247181231775 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002979815437962922 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.002915382935801366 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003368606465907457 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0036279306029124805 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00359268567020384 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.mlp.gate", |
| "sensitivities": { |
| "4": 0.0054580638752668165, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0036926528555341065 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0031067129117824757 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.32.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004197506471731079 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0030901972932042554, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0033126247726613656 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00331102798130208 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0028822653912357055 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003875695178673292 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0038103191618574783 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0038147873856360093 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.mlp.gate", |
| "sensitivities": { |
| "4": 0.00543140152876731, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003964414742464821 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004204695069347508 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0034591444976589023 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.31.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004151555299661898 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.003191108728060499, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003548390481834455 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0036089261120650917 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0034522233545430936 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003841529044924149 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003770305927901063 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0038622769401020682 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.mlp.gate", |
| "sensitivities": { |
| "4": 0.005873478900563593, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004227041160144533 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003324077117819494 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.30.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004752254715034117 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0035896663054397018, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0035883158537520408 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003742401638495115 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0034946068941887156 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004219321068376303 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004259144703003888 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004165746400152178 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.mlp.gate", |
| "sensitivities": { |
| "4": 0.006541496540497367, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0044880598967817304 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0036810627813489796 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.29.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004826595235499553 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.00362065994219544, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.003921926690964028 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0039680428671999834 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0036683329550821022 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0043618389148226315 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004315103074380507 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004349304557157059 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.mlp.gate", |
| "sensitivities": { |
| "4": 0.006787006728700362, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004702370313073819 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0037491704545876323 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.28.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0051801369748621555 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.003750991881436979, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004106528702929306 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004144503269344568 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004088555853134797 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004885390294172491 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00455872366001131 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0049130060215247795 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.mlp.gate", |
| "sensitivities": { |
| "4": 0.007291974553178686, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00555882364763723 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005674804478379277 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004737999302354486 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.27.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005359278341832881 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.004103830414048086, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004563357242053219 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004600786213510825 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004448893041020104 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00518409363576211 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004802735384146217 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005048779246862978 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.mlp.gate", |
| "sensitivities": { |
| "4": 0.008079974698678901, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00514275149422853 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004920122419813803 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.26.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005918499732312436 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.004627415449552548, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004856845213604781 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004881828169648846 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.004638931318671287 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005686172774100366 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0053267386635222165 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005722651037406952 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.mlp.gate", |
| "sensitivities": { |
| "4": 0.009340248730344078, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006284646408554788 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005060244996760351 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.25.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006544227498428275 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.005057090580521617, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005316806911044599 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005873421080953752 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005097836023196578 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006277923326706514 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005662061237671878 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005897963734848115 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.mlp.gate", |
| "sensitivities": { |
| "4": 0.009583479351325272, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006754006913979538 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005299228282334904 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.24.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007472097684512846 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.005364448579105859, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005531614619637064 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005941575296067943 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.005747957458273352 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006524281270685606 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006318350400154789 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006365798185773504 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.mlp.gate", |
| "sensitivities": { |
| "4": 0.010608130556647666, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008208236598875374 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00739130225944488 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006829266402443561 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.23.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007633909597643651 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.00617599854740547, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006110824887097503 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006009446524937327 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006198184875150521 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0070419801292397706 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0064737989838855965 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006815817270156306 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.mlp.gate", |
| "sensitivities": { |
| "4": 0.010461516358191147, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007974229263102947 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006750545023047986 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.22.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009198193957369464 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.006172639778621185, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006532026910766338 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00721957237207486 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006716253750103836 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007162697574434181 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006981045803210388 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007448724689311348 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.mlp.gate", |
| "sensitivities": { |
| "4": 0.011110041804689294, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008667696058788957 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006610841888080661 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.21.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00952446710531755 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.0069478996738325804, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0074366811362172784 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.006943494290074644 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0067551254711967585 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008092523431211399 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007611915270293442 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008078320082859136 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.mlp.gate", |
| "sensitivities": { |
| "4": 0.011131369343881184, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00876020211338376 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007388061237482664 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.20.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0099560285161715 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.007256736600538716, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0075626892648870125 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007552810381942739 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00718522864432695 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008595826366217807 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00823559146859528 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008514910165104084 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.mlp.gate", |
| "sensitivities": { |
| "4": 0.011962555969754854, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009170711811748333 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00948034289952678 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00827406374446582 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.19.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009781325609462025 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.007757856593040439, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008020264431252144 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.007868040115378486 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008096922431529189 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008891638106433675 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008893419241455073 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008583348394798426 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.mlp.gate", |
| "sensitivities": { |
| "4": 0.012067314916445563, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009313053917139769 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00792201085035534 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.18.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010337958544065865 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.00800801339210011, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008354605146450922 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008346605425079664 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00789218878101868 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0096804217416017 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009081653120423047 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00961258050908024 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.mlp.gate", |
| "sensitivities": { |
| "4": 0.012435313129875189, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00969749286499185 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008220665801976187 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.17.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01077947385783773 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.00898167995425562, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008234198758145794 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00856234620732721 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008658537199759545 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009765561121942786 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00955390140976912 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00922697524462516 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.mlp.gate", |
| "sensitivities": { |
| "4": 0.011782195152287992, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010321959528179528 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.008864662149183763 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.16.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010752089612651616 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.009154927955629924, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009189329032475749 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009195887905661948 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009162585886466937 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010390287488310909 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009711528284242377 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010309005903157717 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.mlp.gate", |
| "sensitivities": { |
| "4": 0.013503166526788846, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011138641464640386 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011259745757949228 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009398004641601196 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.15.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0112590963835828 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.009654443934171772, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009863078341974566 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.00967700000910554 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009742690696536252 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010233040569194904 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01022266013508973 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010732679739400433 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.mlp.gate", |
| "sensitivities": { |
| "4": 0.01367227423179429, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01100488739515034 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01022967364891277 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.14.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01245528188883327 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.010582786043717837, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010168050968786702 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01001773014528832 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.009968309891216146 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011261625166904802 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010767707319852585 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010916094227771586 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.mlp.gate", |
| "sensitivities": { |
| "4": 0.01399673714574116, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011629761046303125 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010364441375713795 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.13.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012238817202160135 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.010277952591422945, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010277141176629812 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010602660113363527 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010807378469811132 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012377719894478409 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011131742144546783 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010933864900531868 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.mlp.gate", |
| "sensitivities": { |
| "4": 0.014976006457194066, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011991277909449613 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010463360308979949 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.12.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013422283062633747 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.010844901742530055, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011393746521207504 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011229697129844377 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.010468666019733064 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012109854117928384 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011566853941379426 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011979735451556431 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.mlp.gate", |
| "sensitivities": { |
| "4": 0.014990527405946827, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01275160105433315 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015201601044585308 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012038252505590208 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.11.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013188726123189554 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.011288866294004643, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011611835070652887 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012178075737513913 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.011832942793262191 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012095854433331018 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01274094315400968 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012829635612433776 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.mlp.gate", |
| "sensitivities": { |
| "4": 0.01626087084878236, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013361044965373972 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012351946274672324 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.10.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015012034185929224 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.011678688800505673, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012375947214119757 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012288592877060486 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01256927099893801 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01338555481925141 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012540935538709164 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012916975412129736 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.mlp.gate", |
| "sensitivities": { |
| "4": 0.01662118696064378, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014035383860270182 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012819181167287752 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.9.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015574557250753665 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.012599239038536325, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013240655724075623 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.012855073883353422 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013038330798735842 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013493624350909764 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013379455165704712 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013178177721177539 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.mlp.gate", |
| "sensitivities": { |
| "4": 0.01652742245156939, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014525910800633332 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014052762446226552 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.8.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01691254019776049 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.01293456334193858, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013468108120529601 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013862467652264362 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013406028689738983 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013866370340110734 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014211450363897407 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013236384809715673 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.mlp.gate", |
| "sensitivities": { |
| "4": 0.01609512370972273, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015192151438289633 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01611088200782736 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014883832996323084 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.7.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014775762363569811 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.013842910673702136, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01374226551464138 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015329888090491295 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013457816676236689 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014617457006049031 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014744472709329179 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013872471987269819 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.mlp.gate", |
| "sensitivities": { |
| "4": 0.016978700946007546, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015879868374516565 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014073359741208455 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.6.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016584908017345395 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.013318267408370351, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.013795014693945026 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014510109923624745 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014729883502392719 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014678579987958074 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014759288256755099 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014945248481429493 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.mlp.gate", |
| "sensitivities": { |
| "4": 0.01711107642040588, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014898032823111862 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014382564753759652 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.5.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017053854671151687 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.014288380965202426, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014899879752192646 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015110610518604517 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01435574131513325 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016535441546390455 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015712454575502004 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01456842960518164 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.mlp.gate", |
| "sensitivities": { |
| "4": 0.018017263452444848, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015937547102415312 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015152021961209053 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.4.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01661324539842705 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.01538781247412165, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015848533192183822 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015900589545102168 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.014709234674228355 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.0161793943552766 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.015927003318211064 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016266208646508556 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.mlp.gate", |
| "sensitivities": { |
| "4": 0.018574310747984175, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.self_attn.o_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017166202393127605 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.self_attn.v_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.02210300312920784 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.self_attn.k_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01649043273452359 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.3.self_attn.q_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017937579153416056 |
| }, |
| "param_count": 16777216, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.015848530747462064, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017437926658506814 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016578349061698344 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016357797489035875 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01727279481322815 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016195804220236216 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01615279316320084 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.mlp.gate", |
| "sensitivities": { |
| "4": 0.01900803267684144, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017819210469800357 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017294775665504858 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.2.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.020012144804544125 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.016524562825604033, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016842761435934033 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016998056855906423 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01661954979257037 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017476482074319694 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01659977853220577 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016613098239758983 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.mlp.gate", |
| "sensitivities": { |
| "4": 0.01898803311632946, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01730258480529301 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.018019680787498753 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.1.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01960670785047114 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.shared_expert_gate", |
| "sensitivities": { |
| "4": 0.016866121945592265, |
| "8": 0.0 |
| }, |
| "param_count": 2048, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.shared_expert.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.018234637934559334 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.shared_expert.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017515672865556553 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.shared_expert.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016906839252139132 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.switch_mlp.down_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.018159386556362733 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.switch_mlp.up_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.016372410124555852 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.switch_mlp.gate_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01711415994213894 |
| }, |
| "param_count": 536870912, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.mlp.gate", |
| "sensitivities": { |
| "4": 0.01868543986347504, |
| "8": 0.0 |
| }, |
| "param_count": 1048576, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.linear_attn.out_proj", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.019266100542154163 |
| }, |
| "param_count": 8388608, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.linear_attn.in_proj_ba", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.017209987262807164 |
| }, |
| "param_count": 131072, |
| "metadata": {} |
| }, |
| { |
| "layer_name": "model.layers.0.linear_attn.in_proj_qkvz", |
| "sensitivities": { |
| "4": 0.0, |
| "8": 0.01821280693790565 |
| }, |
| "param_count": 25165824, |
| "metadata": {} |
| } |
| ] |
