[
  {
    "layer_name": "lm_head",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00546584720723331
    },
    "param_count": 311164928,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0001811789904119602,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00087689898039874
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0015896713763747055
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00044233594720329467
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0003866669740091311
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.000519902074302081
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00046726479376957286
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.mlp.gate",
    "sensitivities": {
      "4": 0.0015319683431395485,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0009970010384373988
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0006898588944750372
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.000378026442679887
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.47.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0009673482066621849
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.00043826462933793664,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005482422323742261
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.000585554851568304
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005205812140047783
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005259119376811819
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005636888490698766
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005691807382390834
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.mlp.gate",
    "sensitivities": {
      "4": 0.0016299924997535224,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0013121363129660797
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0004549955301627051
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.46.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001232918316721528
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0004910269999527372,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0006056272814021213
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.000662573301572896
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005977782335927865
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0007479680231578337
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0007716155766199032
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0007242341719878217
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.mlp.gate",
    "sensitivities": {
      "4": 0.002146355339694613,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0011942335452962045
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0005906730202696053
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.45.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0012439823755509376
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0007133473106174885,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0007204592196406642
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0008755783965170849
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0006210746008340114
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.000983773055243849
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0010371999160270207
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0009049700092873536
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.mlp.gate",
    "sensitivities": {
      "4": 0.0024624387491106368,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0015454338014630291
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0007653660904907156
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.44.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0014417152075717847
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0007744313740355816,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0009393947063169131
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0010768429419840686
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0008377297441863144
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0010964083982495747
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0010202171264002875
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0011413340374322918
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.mlp.gate",
    "sensitivities": {
      "4": 0.0033793977951669754,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002702764031710103
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0017084380015148781
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0011191138861856114
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.43.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0014598523788057112
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0009925571054433628,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0011605675669367581
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0018117218254095253
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001172308443832056
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0015623406288796104
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00130574569387439
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0013518400919565465
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.mlp.gate",
    "sensitivities": {
      "4": 0.0029391975913313217,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0017956645048495072
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0011235147382346138
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.42.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002219276189862285
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0011014955040688317,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001253751875386418
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0012832630854973104
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0012258604571494895
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0015417903002041082
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0016429382061081317
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0015326397333410569
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.mlp.gate",
    "sensitivities": {
      "4": 0.0030040774484708286,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0017300844774581492
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0013138200107884284
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.41.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0019365232534861814
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0012782128954616685,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0014563855996433024
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0014926813782949466
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0013535707779131674
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0017617459622366975
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0016695782178430818
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0016951369301144343
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.mlp.gate",
    "sensitivities": {
      "4": 0.0033831604887382127,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002003027924123065
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0014643894125280592
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.40.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002364053131411007
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.001446501639293274,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001575188147398876
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001775789626359862
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0015565581913203157
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0019679781325976364
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001903772181928313
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001958755795688679
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.mlp.gate",
    "sensitivities": {
      "4": 0.0034947048746592677,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0022330831889121328
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0020966503264692924
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0017853173170199927
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.39.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002157975022176591
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0016617042371459927,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.001788420772451597
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0018763207917800173
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0018028927515842952
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00211460222756917
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0020056302140195235
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002035529483691789
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.mlp.gate",
    "sensitivities": {
      "4": 0.003621493626269512,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0020221223191280537
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0017770422612860177
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.38.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0024485136527800933
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0017955446419364307,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00191835029302941
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002024596572785716
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0018729715350976524
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0022996737267628
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0021713513148521693
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0021041337956072916
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.mlp.gate",
    "sensitivities": {
      "4": 0.004091399455016169,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002346425324503798
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0018104871329948462
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.37.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002669086587654116
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0018940380444594969,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002158472236866752
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002250783230313876
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002060351244532891
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002607721893582493
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0025067820267092125
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0025180679634407475
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.mlp.gate",
    "sensitivities": {
      "4": 0.0041634838610965135,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002644306846680896
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002127183021608895
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.36.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003041435554526591
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0021315743118369332,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0023513569540227763
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00261818288709037
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0022630117625036896
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0029103495204860033
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0026857099728658795
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0027523967655724846
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.mlp.gate",
    "sensitivities": {
      "4": 0.004539914623213311,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003231826927124833
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0032008243239639946
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0025570961685540774
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.35.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002896230924913349
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.002297268777814073,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0026993594292434864
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0027452579136782638
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0024244497229422755
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0029562374596328786
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00277543455013074
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00286523503503607
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.mlp.gate",
    "sensitivities": {
      "4": 0.00521997369651217,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003193213837221265
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002630184574324327
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.34.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0036525255200103857
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.002746699678633983,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0027895902336846725
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0029053816663993834
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002501191748403168
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0031272211781470105
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0031219236722487644
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0032260748290961296
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.mlp.gate",
    "sensitivities": {
      "4": 0.005147328270444025,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0033409177121939138
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002774385551068311
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.33.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0038971262256382033
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0028969740136138475,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0031035247181231775
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002979815437962922
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.002915382935801366
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003368606465907457
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0036279306029124805
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00359268567020384
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.mlp.gate",
    "sensitivities": {
      "4": 0.0054580638752668165,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0036926528555341065
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0031067129117824757
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.32.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004197506471731079
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0030901972932042554,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0033126247726613656
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00331102798130208
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0028822653912357055
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003875695178673292
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0038103191618574783
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0038147873856360093
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.mlp.gate",
    "sensitivities": {
      "4": 0.00543140152876731,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003964414742464821
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004204695069347508
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0034591444976589023
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.31.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004151555299661898
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.003191108728060499,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003548390481834455
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0036089261120650917
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0034522233545430936
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003841529044924149
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003770305927901063
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0038622769401020682
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.mlp.gate",
    "sensitivities": {
      "4": 0.005873478900563593,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004227041160144533
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003324077117819494
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.30.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004752254715034117
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0035896663054397018,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0035883158537520408
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003742401638495115
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0034946068941887156
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004219321068376303
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004259144703003888
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004165746400152178
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.mlp.gate",
    "sensitivities": {
      "4": 0.006541496540497367,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0044880598967817304
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0036810627813489796
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.29.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004826595235499553
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.00362065994219544,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.003921926690964028
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0039680428671999834
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0036683329550821022
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0043618389148226315
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004315103074380507
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004349304557157059
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.mlp.gate",
    "sensitivities": {
      "4": 0.006787006728700362,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004702370313073819
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0037491704545876323
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.28.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0051801369748621555
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.003750991881436979,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004106528702929306
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004144503269344568
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004088555853134797
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004885390294172491
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00455872366001131
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0049130060215247795
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.mlp.gate",
    "sensitivities": {
      "4": 0.007291974553178686,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00555882364763723
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005674804478379277
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004737999302354486
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.27.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005359278341832881
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.004103830414048086,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004563357242053219
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004600786213510825
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004448893041020104
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00518409363576211
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004802735384146217
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005048779246862978
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.mlp.gate",
    "sensitivities": {
      "4": 0.008079974698678901,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00514275149422853
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004920122419813803
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.26.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005918499732312436
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.004627415449552548,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004856845213604781
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004881828169648846
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.004638931318671287
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005686172774100366
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0053267386635222165
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005722651037406952
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.mlp.gate",
    "sensitivities": {
      "4": 0.009340248730344078,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006284646408554788
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005060244996760351
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.25.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006544227498428275
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.005057090580521617,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005316806911044599
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005873421080953752
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005097836023196578
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006277923326706514
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005662061237671878
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005897963734848115
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.mlp.gate",
    "sensitivities": {
      "4": 0.009583479351325272,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006754006913979538
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005299228282334904
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.24.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007472097684512846
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.005364448579105859,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005531614619637064
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005941575296067943
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.005747957458273352
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006524281270685606
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006318350400154789
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006365798185773504
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.mlp.gate",
    "sensitivities": {
      "4": 0.010608130556647666,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008208236598875374
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00739130225944488
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006829266402443561
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.23.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007633909597643651
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.00617599854740547,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006110824887097503
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006009446524937327
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006198184875150521
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0070419801292397706
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0064737989838855965
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006815817270156306
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.mlp.gate",
    "sensitivities": {
      "4": 0.010461516358191147,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007974229263102947
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006750545023047986
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.22.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009198193957369464
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.006172639778621185,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006532026910766338
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00721957237207486
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006716253750103836
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007162697574434181
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006981045803210388
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007448724689311348
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.mlp.gate",
    "sensitivities": {
      "4": 0.011110041804689294,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008667696058788957
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006610841888080661
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.21.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00952446710531755
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.0069478996738325804,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0074366811362172784
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.006943494290074644
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0067551254711967585
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008092523431211399
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007611915270293442
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008078320082859136
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.mlp.gate",
    "sensitivities": {
      "4": 0.011131369343881184,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00876020211338376
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007388061237482664
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.20.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0099560285161715
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.007256736600538716,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0075626892648870125
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007552810381942739
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00718522864432695
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008595826366217807
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00823559146859528
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008514910165104084
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.mlp.gate",
    "sensitivities": {
      "4": 0.011962555969754854,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009170711811748333
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00948034289952678
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00827406374446582
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.19.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009781325609462025
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.007757856593040439,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008020264431252144
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.007868040115378486
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008096922431529189
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008891638106433675
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008893419241455073
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008583348394798426
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.mlp.gate",
    "sensitivities": {
      "4": 0.012067314916445563,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009313053917139769
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00792201085035534
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.18.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010337958544065865
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.00800801339210011,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008354605146450922
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008346605425079664
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00789218878101868
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0096804217416017
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009081653120423047
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00961258050908024
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.mlp.gate",
    "sensitivities": {
      "4": 0.012435313129875189,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00969749286499185
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008220665801976187
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.17.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01077947385783773
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.00898167995425562,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008234198758145794
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00856234620732721
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008658537199759545
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009765561121942786
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00955390140976912
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00922697524462516
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.mlp.gate",
    "sensitivities": {
      "4": 0.011782195152287992,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010321959528179528
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.008864662149183763
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.16.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010752089612651616
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.009154927955629924,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009189329032475749
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009195887905661948
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009162585886466937
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010390287488310909
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009711528284242377
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010309005903157717
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.mlp.gate",
    "sensitivities": {
      "4": 0.013503166526788846,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011138641464640386
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011259745757949228
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009398004641601196
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.15.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0112590963835828
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.009654443934171772,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009863078341974566
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.00967700000910554
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009742690696536252
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010233040569194904
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01022266013508973
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010732679739400433
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.mlp.gate",
    "sensitivities": {
      "4": 0.01367227423179429,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01100488739515034
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01022967364891277
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.14.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01245528188883327
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.010582786043717837,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010168050968786702
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01001773014528832
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.009968309891216146
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011261625166904802
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010767707319852585
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010916094227771586
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.mlp.gate",
    "sensitivities": {
      "4": 0.01399673714574116,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011629761046303125
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010364441375713795
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.13.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012238817202160135
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.010277952591422945,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010277141176629812
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010602660113363527
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010807378469811132
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012377719894478409
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011131742144546783
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010933864900531868
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.mlp.gate",
    "sensitivities": {
      "4": 0.014976006457194066,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011991277909449613
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010463360308979949
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.12.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013422283062633747
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.010844901742530055,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011393746521207504
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011229697129844377
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.010468666019733064
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012109854117928384
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011566853941379426
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011979735451556431
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.mlp.gate",
    "sensitivities": {
      "4": 0.014990527405946827,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01275160105433315
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015201601044585308
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012038252505590208
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.11.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013188726123189554
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.011288866294004643,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011611835070652887
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012178075737513913
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.011832942793262191
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012095854433331018
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01274094315400968
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012829635612433776
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.mlp.gate",
    "sensitivities": {
      "4": 0.01626087084878236,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013361044965373972
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012351946274672324
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.10.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015012034185929224
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.011678688800505673,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012375947214119757
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012288592877060486
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01256927099893801
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01338555481925141
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012540935538709164
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012916975412129736
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.mlp.gate",
    "sensitivities": {
      "4": 0.01662118696064378,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014035383860270182
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012819181167287752
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.9.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015574557250753665
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.012599239038536325,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013240655724075623
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.012855073883353422
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013038330798735842
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013493624350909764
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013379455165704712
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013178177721177539
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.mlp.gate",
    "sensitivities": {
      "4": 0.01652742245156939,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014525910800633332
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014052762446226552
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.8.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01691254019776049
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.01293456334193858,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013468108120529601
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013862467652264362
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013406028689738983
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013866370340110734
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014211450363897407
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013236384809715673
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.mlp.gate",
    "sensitivities": {
      "4": 0.01609512370972273,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015192151438289633
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01611088200782736
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014883832996323084
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.7.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014775762363569811
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.013842910673702136,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01374226551464138
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015329888090491295
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013457816676236689
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014617457006049031
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014744472709329179
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013872471987269819
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.mlp.gate",
    "sensitivities": {
      "4": 0.016978700946007546,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015879868374516565
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014073359741208455
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.6.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016584908017345395
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.013318267408370351,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.013795014693945026
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014510109923624745
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014729883502392719
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014678579987958074
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014759288256755099
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014945248481429493
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.mlp.gate",
    "sensitivities": {
      "4": 0.01711107642040588,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014898032823111862
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014382564753759652
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.5.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017053854671151687
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.014288380965202426,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014899879752192646
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015110610518604517
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01435574131513325
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016535441546390455
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015712454575502004
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01456842960518164
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.mlp.gate",
    "sensitivities": {
      "4": 0.018017263452444848,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015937547102415312
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015152021961209053
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.4.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01661324539842705
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.01538781247412165,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015848533192183822
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015900589545102168
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.014709234674228355
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.0161793943552766
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.015927003318211064
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016266208646508556
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.mlp.gate",
    "sensitivities": {
      "4": 0.018574310747984175,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.self_attn.o_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017166202393127605
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.self_attn.v_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.02210300312920784
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.self_attn.k_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01649043273452359
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.3.self_attn.q_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017937579153416056
    },
    "param_count": 16777216,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.015848530747462064,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017437926658506814
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016578349061698344
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016357797489035875
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01727279481322815
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016195804220236216
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01615279316320084
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.mlp.gate",
    "sensitivities": {
      "4": 0.01900803267684144,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017819210469800357
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017294775665504858
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.2.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.020012144804544125
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.016524562825604033,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016842761435934033
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016998056855906423
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01661954979257037
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017476482074319694
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01659977853220577
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016613098239758983
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.mlp.gate",
    "sensitivities": {
      "4": 0.01898803311632946,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01730258480529301
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.018019680787498753
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.1.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01960670785047114
    },
    "param_count": 25165824,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.shared_expert_gate",
    "sensitivities": {
      "4": 0.016866121945592265,
      "8": 0.0
    },
    "param_count": 2048,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.shared_expert.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.018234637934559334
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.shared_expert.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017515672865556553
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.shared_expert.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016906839252139132
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.switch_mlp.down_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.018159386556362733
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.switch_mlp.up_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.016372410124555852
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.switch_mlp.gate_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01711415994213894
    },
    "param_count": 536870912,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.mlp.gate",
    "sensitivities": {
      "4": 0.01868543986347504,
      "8": 0.0
    },
    "param_count": 1048576,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.linear_attn.out_proj",
    "sensitivities": {
      "4": 0.0,
      "8": 0.019266100542154163
    },
    "param_count": 8388608,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.linear_attn.in_proj_ba",
    "sensitivities": {
      "4": 0.0,
      "8": 0.017209987262807164
    },
    "param_count": 131072,
    "metadata": {}
  },
  {
    "layer_name": "model.layers.0.linear_attn.in_proj_qkvz",
    "sensitivities": {
      "4": 0.0,
      "8": 0.01821280693790565
    },
    "param_count": 25165824,
    "metadata": {}
  }
]