Upload bayesian_features.json

bayesian_features.json

@@ -0,0 +1,847 @@
+{
+    "description": "Bayesian scalar features and model layer shapes",
+    "base_predictions": {
+        "model.layers.0.self_attn.q_proj.weight": -2.8159893190604635e-06,
+        "model.layers.0.self_attn.k_proj.weight": 9.504846093477681e-05,
+        "model.layers.0.self_attn.v_proj.weight": -1.5374733266071416e-05,
+        "model.layers.0.self_attn.o_proj.weight": -9.122464689426124e-06,
+        "model.layers.0.mlp.gate_proj.weight": 8.1924008554779e-06,
+        "model.layers.0.mlp.up_proj.weight": -1.057311692420626e-05,
+        "model.layers.0.mlp.down_proj.weight": 1.7458359025113168e-06,
+        "model.layers.1.self_attn.q_proj.weight": -7.108108547981828e-05,
+        "model.layers.1.self_attn.k_proj.weight": -9.288229193771258e-05,
+        "model.layers.1.self_attn.v_proj.weight": -2.9976623409311287e-05,
+        "model.layers.1.self_attn.o_proj.weight": -1.5204574083327316e-05,
+        "model.layers.1.mlp.gate_proj.weight": 2.9821971111232415e-05,
+        "model.layers.1.mlp.up_proj.weight": 6.262406714085955e-06,
+        "model.layers.1.mlp.down_proj.weight": -8.086964953690767e-06,
+        "model.layers.2.self_attn.q_proj.weight": 3.9845537685323507e-05,
+        "model.layers.2.self_attn.k_proj.weight": 6.523875526909251e-06,
+        "model.layers.2.self_attn.v_proj.weight": 1.3346652849577367e-06,
+        "model.layers.2.self_attn.o_proj.weight": 2.167806087527424e-05,
+        "model.layers.2.mlp.gate_proj.weight": -4.643025476980256e-06,
+        "model.layers.2.mlp.up_proj.weight": 2.54216365647153e-06,
+        "model.layers.2.mlp.down_proj.weight": 2.3367899757431587e-06,
+        "model.layers.3.self_attn.q_proj.weight": -4.690831701736897e-05,
+        "model.layers.3.self_attn.k_proj.weight": -8.759165211813524e-05,
+        "model.layers.3.self_attn.v_proj.weight": -2.8568936613737606e-05,
+        "model.layers.3.self_attn.o_proj.weight": 5.937995410931762e-07,
+        "model.layers.3.mlp.gate_proj.weight": 3.224427200620994e-05,
+        "model.layers.3.mlp.up_proj.weight": -2.6833818083105143e-06,
+        "model.layers.3.mlp.down_proj.weight": -2.5323888621642254e-06,
+        "model.layers.4.self_attn.q_proj.weight": -4.194932625978254e-05,
+        "model.layers.4.self_attn.k_proj.weight": 1.487184988491208e-07,
+        "model.layers.4.self_attn.v_proj.weight": 3.9514063246315345e-05,
+        "model.layers.4.self_attn.o_proj.weight": 2.851296130756964e-06,
+        "model.layers.4.mlp.gate_proj.weight": -4.9818558181868866e-05,
+        "model.layers.4.mlp.up_proj.weight": 3.4667527870624326e-06,
+        "model.layers.4.mlp.down_proj.weight": -6.655030233559955e-07,
+        "model.layers.5.self_attn.q_proj.weight": 5.081994095235132e-05,
+        "model.layers.5.self_attn.k_proj.weight": -9.06619243323803e-05,
+        "model.layers.5.self_attn.v_proj.weight": 1.2527189937827643e-05,
+        "model.layers.5.self_attn.o_proj.weight": 6.323883553704945e-06,
+        "model.layers.5.mlp.gate_proj.weight": 2.5669101887615398e-05,
+        "model.layers.5.mlp.up_proj.weight": 1.7728603779687546e-05,
+        "model.layers.5.mlp.down_proj.weight": 1.8511283997213468e-05,
+        "model.layers.6.self_attn.q_proj.weight": -8.783595694694668e-05,
+        "model.layers.6.self_attn.k_proj.weight": 3.197391561116092e-05,
+        "model.layers.6.self_attn.v_proj.weight": 6.0727928939741105e-05,
+        "model.layers.6.self_attn.o_proj.weight": -1.858077439464978e-06,
+        "model.layers.6.mlp.gate_proj.weight": 7.373692369583296e-06,
+        "model.layers.6.mlp.up_proj.weight": -7.5099742389284074e-06,
+        "model.layers.6.mlp.down_proj.weight": -8.91167474037502e-06,
+        "model.layers.7.self_attn.q_proj.weight": 3.0392282496904954e-05,
+        "model.layers.7.self_attn.k_proj.weight": 4.690723289968446e-05,
+        "model.layers.7.self_attn.v_proj.weight": 5.4113793339638505e-06,
+        "model.layers.7.self_attn.o_proj.weight": -1.6763269741204567e-05,
+        "model.layers.7.mlp.gate_proj.weight": 5.1075414376100525e-05,
+        "model.layers.7.mlp.up_proj.weight": 1.720458567433525e-05,
+        "model.layers.7.mlp.down_proj.weight": 1.8199965779785998e-05,
+        "model.layers.8.self_attn.q_proj.weight": 8.28661029572686e-07,
+        "model.layers.8.self_attn.k_proj.weight": 0.00021502703020814806,
+        "model.layers.8.self_attn.v_proj.weight": -8.60888758325018e-05,
+        "model.layers.8.self_attn.o_proj.weight": -1.3623013728647493e-05,
+        "model.layers.8.mlp.gate_proj.weight": -3.924618067685515e-05,
+        "model.layers.8.mlp.up_proj.weight": 9.285383271162573e-07,
+        "model.layers.8.mlp.down_proj.weight": 3.677302629512269e-06,
+        "model.layers.9.self_attn.q_proj.weight": 1.3768755707133096e-05,
+        "model.layers.9.self_attn.k_proj.weight": 4.0544469811720774e-05,
+        "model.layers.9.self_attn.v_proj.weight": 2.7841797418659553e-05,
+        "model.layers.9.self_attn.o_proj.weight": 2.4110040612868033e-05,
+        "model.layers.9.mlp.gate_proj.weight": 1.5645458915969357e-05,
+        "model.layers.9.mlp.up_proj.weight": -9.087954708775214e-07,
+        "model.layers.9.mlp.down_proj.weight": 1.7958354874281213e-06,
+        "model.layers.10.self_attn.q_proj.weight": -3.053919499507174e-05,
+        "model.layers.10.self_attn.k_proj.weight": -6.580522313015535e-05,
+        "model.layers.10.self_attn.v_proj.weight": 4.194457869743928e-05,
+        "model.layers.10.self_attn.o_proj.weight": 6.235394266695948e-06,
+        "model.layers.10.mlp.gate_proj.weight": 2.899049832194578e-05,
+        "model.layers.10.mlp.up_proj.weight": 4.082730811205693e-06,
+        "model.layers.10.mlp.down_proj.weight": 6.072628821129911e-06,
+        "model.layers.11.self_attn.q_proj.weight": -2.6586967578623444e-05,
+        "model.layers.11.self_attn.k_proj.weight": 2.1866351289645536e-06,
+        "model.layers.11.self_attn.v_proj.weight": -5.336904359865002e-05,
+        "model.layers.11.self_attn.o_proj.weight": 9.377148671774194e-06,
+        "model.layers.11.mlp.gate_proj.weight": 9.416668763151392e-05,
+        "model.layers.11.mlp.up_proj.weight": -2.3078193862602348e-06,
+        "model.layers.11.mlp.down_proj.weight": -2.0557056359393755e-06,
+        "model.layers.12.self_attn.q_proj.weight": -6.7836017478839494e-06,
+        "model.layers.12.self_attn.k_proj.weight": -5.823265837534564e-06,
+        "model.layers.12.self_attn.v_proj.weight": 8.047290975810029e-06,
+        "model.layers.12.self_attn.o_proj.weight": -1.851466549851466e-05,
+        "model.layers.12.mlp.gate_proj.weight": 6.349570321617648e-05,
+        "model.layers.12.mlp.up_proj.weight": 6.288545591814909e-06,
+        "model.layers.12.mlp.down_proj.weight": 7.484853995265439e-06,
+        "model.layers.13.self_attn.q_proj.weight": 3.590376581996679e-05,
+        "model.layers.13.self_attn.k_proj.weight": 2.0736935766763054e-05,
+        "model.layers.13.self_attn.v_proj.weight": 0.00012856072862632573,
+        "model.layers.13.self_attn.o_proj.weight": -1.2794000213034451e-05,
+        "model.layers.13.mlp.gate_proj.weight": 7.19341478543356e-05,
+        "model.layers.13.mlp.up_proj.weight": 1.1427628123783506e-06,
+        "model.layers.13.mlp.down_proj.weight": 1.028338161290776e-07,
+        "model.layers.14.self_attn.q_proj.weight": -1.7676582501735538e-05,
+        "model.layers.14.self_attn.k_proj.weight": 0.00010539514914853498,
+        "model.layers.14.self_attn.v_proj.weight": -5.02027032780461e-05,
+        "model.layers.14.self_attn.o_proj.weight": 1.0813992957992014e-05,
+        "model.layers.14.mlp.gate_proj.weight": 0.00011480705870781094,
+        "model.layers.14.mlp.up_proj.weight": 5.9115232033946086e-06,
+        "model.layers.14.mlp.down_proj.weight": 1.5825449963813298e-06,
+        "model.layers.15.self_attn.q_proj.weight": -3.523179839248769e-05,
+        "model.layers.15.self_attn.k_proj.weight": 0.00011090694897575304,
+        "model.layers.15.self_attn.v_proj.weight": -7.332745553867426e-06,
+        "model.layers.15.self_attn.o_proj.weight": 3.1373001547763124e-06,
+        "model.layers.15.mlp.gate_proj.weight": 8.899769454728812e-05,
+        "model.layers.15.mlp.up_proj.weight": 7.195218131528236e-06,
+        "model.layers.15.mlp.down_proj.weight": 3.6597518828784814e-06,
+        "model.layers.16.self_attn.q_proj.weight": 5.306202365318313e-05,
+        "model.layers.16.self_attn.k_proj.weight": -3.131839548586868e-05,
+        "model.layers.16.self_attn.v_proj.weight": -1.7601294530322775e-05,
+        "model.layers.16.self_attn.o_proj.weight": -8.281715054181404e-06,
+        "model.layers.16.mlp.gate_proj.weight": 9.49468267208431e-06,
+        "model.layers.16.mlp.up_proj.weight": -1.163188699138118e-05,
+        "model.layers.16.mlp.down_proj.weight": 1.4718655620526988e-05,
+        "model.layers.17.self_attn.q_proj.weight": -8.973433068604209e-06,
+        "model.layers.17.self_attn.k_proj.weight": 3.7808047636644915e-05,
+        "model.layers.17.self_attn.v_proj.weight": -6.911561649758369e-05,
+        "model.layers.17.self_attn.o_proj.weight": -9.579152902006172e-06,
+        "model.layers.17.mlp.gate_proj.weight": -0.00010830343671841547,
+        "model.layers.17.mlp.up_proj.weight": -1.2857419278589077e-05,
+        "model.layers.17.mlp.down_proj.weight": 1.2142132845838205e-06,
+        "model.layers.18.self_attn.q_proj.weight": -1.0464565093570855e-05,
+        "model.layers.18.self_attn.k_proj.weight": -4.644897489924915e-05,
+        "model.layers.18.self_attn.v_proj.weight": -5.738062100135721e-05,
+        "model.layers.18.self_attn.o_proj.weight": -5.787829650216736e-06,
+        "model.layers.18.mlp.gate_proj.weight": -3.279325028415769e-05,
+        "model.layers.18.mlp.up_proj.weight": -1.1034922863473184e-05,
+        "model.layers.18.mlp.down_proj.weight": 1.7817993693824974e-06,
+        "model.layers.19.self_attn.q_proj.weight": -6.523631782329176e-06,
+        "model.layers.19.self_attn.k_proj.weight": -2.6879666620516218e-05,
+        "model.layers.19.self_attn.v_proj.weight": -7.250291673699394e-05,
+        "model.layers.19.self_attn.o_proj.weight": -1.2581798728206195e-05,
+        "model.layers.19.mlp.gate_proj.weight": -3.895289410138503e-05,
+        "model.layers.19.mlp.up_proj.weight": 1.7011852833093144e-05,
+        "model.layers.19.mlp.down_proj.weight": 9.61935484156129e-07,
+        "model.layers.20.self_attn.q_proj.weight": 2.036054183918168e-06,
+        "model.layers.20.self_attn.k_proj.weight": 6.745044811395928e-05,
+        "model.layers.20.self_attn.v_proj.weight": 0.00014138614642433822,
+        "model.layers.20.self_attn.o_proj.weight": 9.64260379987536e-06,
+        "model.layers.20.mlp.gate_proj.weight": -1.8843405996449292e-05,
+        "model.layers.20.mlp.up_proj.weight": -4.28432940680068e-06,
+        "model.layers.20.mlp.down_proj.weight": -1.2949315532750916e-05,
+        "model.layers.21.self_attn.q_proj.weight": -3.101234142377507e-06,
+        "model.layers.21.self_attn.k_proj.weight": 2.5284974981332198e-05,
+        "model.layers.21.self_attn.v_proj.weight": -7.869380351621658e-05,
+        "model.layers.21.self_attn.o_proj.weight": -1.1483875823614653e-05,
+        "model.layers.21.mlp.gate_proj.weight": 3.9686776290182024e-05,
+        "model.layers.21.mlp.up_proj.weight": -1.952264483406907e-06,
+        "model.layers.21.mlp.down_proj.weight": -5.479515493789222e-06,
+        "model.layers.22.self_attn.q_proj.weight": -1.8174790966440924e-05,
+        "model.layers.22.self_attn.k_proj.weight": -8.149358473019674e-05,
+        "model.layers.22.self_attn.v_proj.weight": -5.3864387155044824e-05,
+        "model.layers.22.self_attn.o_proj.weight": 5.406914169725496e-06,
+        "model.layers.22.mlp.gate_proj.weight": -1.7827629562816583e-05,
+        "model.layers.22.mlp.up_proj.weight": -1.5168808204180095e-05,
+        "model.layers.22.mlp.down_proj.weight": 8.480022870571702e-07,
+        "model.layers.23.self_attn.q_proj.weight": 1.214700114360312e-05,
+        "model.layers.23.self_attn.k_proj.weight": 2.0788542315131053e-05,
+        "model.layers.23.self_attn.v_proj.weight": 6.853179365862161e-05,
+        "model.layers.23.self_attn.o_proj.weight": -5.629223664982419e-07,
+        "model.layers.23.mlp.gate_proj.weight": 8.198077557608485e-05,
+        "model.layers.23.mlp.up_proj.weight": -1.3765425137535203e-05,
+        "model.layers.23.mlp.down_proj.weight": -4.1605867409089115e-06
+    },
+    "layer_shapes": {
+        "model.layers.0.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.0.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.0.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.0.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.0.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.0.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.0.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.1.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.1.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.1.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.1.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.1.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.1.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.1.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.2.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.2.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.2.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.2.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.2.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.2.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.2.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.3.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.3.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.3.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.3.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.3.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.3.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.3.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.4.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.4.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.4.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.4.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.4.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.4.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.4.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.5.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.5.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.5.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.5.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.5.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.5.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.5.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.6.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.6.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.6.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.6.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.6.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.6.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.6.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.7.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.7.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.7.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.7.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.7.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.7.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.7.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.8.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.8.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.8.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.8.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.8.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.8.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.8.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.9.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.9.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.9.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.9.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.9.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.9.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.9.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.10.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.10.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.10.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.10.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.10.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.10.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.10.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.11.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.11.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.11.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.11.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.11.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.11.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.11.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.12.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.12.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.12.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.12.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.12.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.12.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.12.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.13.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.13.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.13.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.13.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.13.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.13.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.13.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.14.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.14.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.14.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.14.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.14.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.14.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.14.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.15.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.15.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.15.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.15.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.15.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.15.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.15.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.16.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.16.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.16.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.16.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.16.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.16.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.16.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.17.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.17.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.17.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.17.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.17.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.17.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.17.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.18.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.18.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.18.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.18.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.18.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.18.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.18.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.19.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.19.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.19.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.19.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.19.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.19.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.19.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.20.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.20.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.20.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.20.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.20.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.20.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.20.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.21.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.21.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.21.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.21.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.21.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.21.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.21.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.22.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.22.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.22.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.22.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.22.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.22.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.22.mlp.down_proj.weight": [
+            896,
+            4864
+        ],
+        "model.layers.23.self_attn.q_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.23.self_attn.k_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.23.self_attn.v_proj.weight": [
+            128,
+            896
+        ],
+        "model.layers.23.self_attn.o_proj.weight": [
+            896,
+            896
+        ],
+        "model.layers.23.mlp.gate_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.23.mlp.up_proj.weight": [
+            4864,
+            896
+        ],
+        "model.layers.23.mlp.down_proj.weight": [
+            896,
+            4864
+        ]
+    }
+}