| { |
| "rank": 256, |
| "max_residual": 0.05, |
| "residuals": { |
| "model.language_model.layers.40.linear_attn.in_proj_a.weight": { |
| "residual": 0.002146052196621895, |
| "expected_noise": 0.012974443843666715 |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024941368028521538, |
| "expected_noise": 0.012081978087675636 |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.01444031111896038, |
| "expected_noise": 0.014606946275233464 |
| }, |
| "model.language_model.layers.40.linear_attn.in_proj_z.weight": { |
| "residual": 0.011644049547612667, |
| "expected_noise": 0.011715060297014478 |
| }, |
| "model.language_model.layers.40.linear_attn.out_proj.weight": { |
| "residual": 0.023646950721740723, |
| "expected_noise": 0.02451279331265909 |
| }, |
| "model.language_model.layers.40.mlp.down_proj.weight": { |
| "residual": 0.025049719959497452, |
| "expected_noise": 0.02567986889698744 |
| }, |
| "model.language_model.layers.40.mlp.gate_proj.weight": { |
| "residual": 0.011802067048847675, |
| "expected_noise": 0.011712125321091709 |
| }, |
| "model.language_model.layers.40.mlp.up_proj.weight": { |
| "residual": 0.018294109031558037, |
| "expected_noise": 0.018593062788491467 |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023987703025341034, |
| "expected_noise": 0.015031720988324622 |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_b.weight": { |
| "residual": 0.002487862017005682, |
| "expected_noise": 0.010877011116313404 |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.016599491238594055, |
| "expected_noise": 0.01681164873421441 |
| }, |
| "model.language_model.layers.41.linear_attn.in_proj_z.weight": { |
| "residual": 0.015272041782736778, |
| "expected_noise": 0.01562853986274499 |
| }, |
| "model.language_model.layers.41.linear_attn.out_proj.weight": { |
| "residual": 0.02833371050655842, |
| "expected_noise": 0.029499450273880207 |
| }, |
| "model.language_model.layers.41.mlp.down_proj.weight": { |
| "residual": 0.029693936929106712, |
| "expected_noise": 0.030523855890532622 |
| }, |
| "model.language_model.layers.41.mlp.gate_proj.weight": { |
| "residual": 0.014496462419629097, |
| "expected_noise": 0.0145778493576875 |
| }, |
| "model.language_model.layers.41.mlp.up_proj.weight": { |
| "residual": 0.019872894510626793, |
| "expected_noise": 0.020249397125064576 |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_a.weight": { |
| "residual": 0.002244287636131048, |
| "expected_noise": 0.012608820256705932 |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024181529879570007, |
| "expected_noise": 0.009967008221082102 |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.020915284752845764, |
| "expected_noise": 0.02146663214507366 |
| }, |
| "model.language_model.layers.42.linear_attn.in_proj_z.weight": { |
| "residual": 0.014570524916052818, |
| "expected_noise": 0.014861297978906701 |
| }, |
| "model.language_model.layers.42.linear_attn.out_proj.weight": { |
| "residual": 0.030799880623817444, |
| "expected_noise": 0.03211886467913097 |
| }, |
| "model.language_model.layers.42.mlp.down_proj.weight": { |
| "residual": 0.030454199761152267, |
| "expected_noise": 0.031328033541941205 |
| }, |
| "model.language_model.layers.42.mlp.gate_proj.weight": { |
| "residual": 0.016223207116127014, |
| "expected_noise": 0.01640663208177938 |
| }, |
| "model.language_model.layers.42.mlp.up_proj.weight": { |
| "residual": 0.020855600014328957, |
| "expected_noise": 0.02127874308905236 |
| }, |
| "model.language_model.layers.43.mlp.down_proj.weight": { |
| "residual": 0.032823145389556885, |
| "expected_noise": 0.03378377377276178 |
| }, |
| "model.language_model.layers.43.mlp.gate_proj.weight": { |
| "residual": 0.019350314512848854, |
| "expected_noise": 0.01969872596677415 |
| }, |
| "model.language_model.layers.43.mlp.up_proj.weight": { |
| "residual": 0.02309340238571167, |
| "expected_noise": 0.023623713819061757 |
| }, |
| "model.language_model.layers.43.self_attn.k_proj.weight": { |
| "residual": 0.02066306583583355, |
| "expected_noise": 0.02434440532880551 |
| }, |
| "model.language_model.layers.43.self_attn.o_proj.weight": { |
| "residual": 0.03257371485233307, |
| "expected_noise": 0.033977171990374706 |
| }, |
| "model.language_model.layers.43.self_attn.q_proj.weight": { |
| "residual": 0.02321186102926731, |
| "expected_noise": 0.02383428591468695 |
| }, |
| "model.language_model.layers.43.self_attn.v_proj.weight": { |
| "residual": 0.02583351917564869, |
| "expected_noise": 0.030454820453565307 |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_a.weight": { |
| "residual": 0.002216341206803918, |
| "expected_noise": 0.012409744532101868 |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_b.weight": { |
| "residual": 0.002490745158866048, |
| "expected_noise": 0.009856438750009123 |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.025673801079392433, |
| "expected_noise": 0.02647559578788941 |
| }, |
| "model.language_model.layers.44.linear_attn.in_proj_z.weight": { |
| "residual": 0.017309727147221565, |
| "expected_noise": 0.0177499164476529 |
| }, |
| "model.language_model.layers.44.linear_attn.out_proj.weight": { |
| "residual": 0.03230440989136696, |
| "expected_noise": 0.033712908833213345 |
| }, |
| "model.language_model.layers.44.mlp.down_proj.weight": { |
| "residual": 0.031004754826426506, |
| "expected_noise": 0.03189631559201829 |
| }, |
| "model.language_model.layers.44.mlp.gate_proj.weight": { |
| "residual": 0.01922808773815632, |
| "expected_noise": 0.01957301177624235 |
| }, |
| "model.language_model.layers.44.mlp.up_proj.weight": { |
| "residual": 0.022256333380937576, |
| "expected_noise": 0.022746449441290693 |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_a.weight": { |
| "residual": 0.0022388342767953873, |
| "expected_noise": 0.014264842383568442 |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_b.weight": { |
| "residual": 0.002298386301845312, |
| "expected_noise": 0.010848777437960757 |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.024941040202975273, |
| "expected_noise": 0.02570060683459132 |
| }, |
| "model.language_model.layers.45.linear_attn.in_proj_z.weight": { |
| "residual": 0.01844269037246704, |
| "expected_noise": 0.019001911225722905 |
| }, |
| "model.language_model.layers.45.linear_attn.out_proj.weight": { |
| "residual": 0.029758809134364128, |
| "expected_noise": 0.031028814902908496 |
| }, |
| "model.language_model.layers.45.mlp.down_proj.weight": { |
| "residual": 0.02979564666748047, |
| "expected_noise": 0.0306394651651357 |
| }, |
| "model.language_model.layers.45.mlp.gate_proj.weight": { |
| "residual": 0.018665647134184837, |
| "expected_noise": 0.0189769975480935 |
| }, |
| "model.language_model.layers.45.mlp.up_proj.weight": { |
| "residual": 0.02305259369313717, |
| "expected_noise": 0.02357369985563593 |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_a.weight": { |
| "residual": 0.0021862995345145464, |
| "expected_noise": 0.0136549115320459 |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_b.weight": { |
| "residual": 0.002302567008882761, |
| "expected_noise": 0.011588226213113176 |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02518370747566223, |
| "expected_noise": 0.02595827159156582 |
| }, |
| "model.language_model.layers.46.linear_attn.in_proj_z.weight": { |
| "residual": 0.016484098508954048, |
| "expected_noise": 0.01678300866186771 |
| }, |
| "model.language_model.layers.46.linear_attn.out_proj.weight": { |
| "residual": 0.030089348554611206, |
| "expected_noise": 0.03137046317394857 |
| }, |
| "model.language_model.layers.46.mlp.down_proj.weight": { |
| "residual": 0.02790791355073452, |
| "expected_noise": 0.0286551141854796 |
| }, |
| "model.language_model.layers.46.mlp.gate_proj.weight": { |
| "residual": 0.018145939335227013, |
| "expected_noise": 0.018439166107186968 |
| }, |
| "model.language_model.layers.46.mlp.up_proj.weight": { |
| "residual": 0.02233639545738697, |
| "expected_noise": 0.022834483731352197 |
| }, |
| "model.language_model.layers.47.mlp.down_proj.weight": { |
| "residual": 0.023689016699790955, |
| "expected_noise": 0.024249328031035355 |
| }, |
| "model.language_model.layers.47.mlp.gate_proj.weight": { |
| "residual": 0.016771014779806137, |
| "expected_noise": 0.01696912237101654 |
| }, |
| "model.language_model.layers.47.mlp.up_proj.weight": { |
| "residual": 0.02183159440755844, |
| "expected_noise": 0.022308856835644132 |
| }, |
| "model.language_model.layers.47.self_attn.k_proj.weight": { |
| "residual": 0.021325351670384407, |
| "expected_noise": 0.025119833629998575 |
| }, |
| "model.language_model.layers.47.self_attn.o_proj.weight": { |
| "residual": 0.032491233199834824, |
| "expected_noise": 0.03391210515796512 |
| }, |
| "model.language_model.layers.47.self_attn.q_proj.weight": { |
| "residual": 0.023718250915408134, |
| "expected_noise": 0.024367478366397438 |
| }, |
| "model.language_model.layers.47.self_attn.v_proj.weight": { |
| "residual": 0.033579420298337936, |
| "expected_noise": 0.03972490845880818 |
| }, |
| "model.language_model.layers.48.linear_attn.in_proj_a.weight": { |
| "residual": 0.002346278168261051, |
| "expected_noise": 0.011280545329459832 |
| }, |
| "model.language_model.layers.48.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022730999626219273, |
| "expected_noise": 0.009259874055325875 |
| }, |
| "model.language_model.layers.48.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.025525910779833794, |
| "expected_noise": 0.026318426907857614 |
| }, |
| "model.language_model.layers.48.linear_attn.in_proj_z.weight": { |
| "residual": 0.014913615770637989, |
| "expected_noise": 0.01523274634308516 |
| }, |
| "model.language_model.layers.48.linear_attn.out_proj.weight": { |
| "residual": 0.025087514892220497, |
| "expected_noise": 0.0260888624832962 |
| }, |
| "model.language_model.layers.48.mlp.down_proj.weight": { |
| "residual": 0.02170288935303688, |
| "expected_noise": 0.02215326645805858 |
| }, |
| "model.language_model.layers.48.mlp.gate_proj.weight": { |
| "residual": 0.014320943504571915, |
| "expected_noise": 0.0143976298961414 |
| }, |
| "model.language_model.layers.48.mlp.up_proj.weight": { |
| "residual": 0.02098502404987812, |
| "expected_noise": 0.021421334949477722 |
| }, |
| "model.language_model.layers.49.linear_attn.in_proj_a.weight": { |
| "residual": 0.0024136558640748262, |
| "expected_noise": 0.018704550434213242 |
| }, |
| "model.language_model.layers.49.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024829739704728127, |
| "expected_noise": 0.010717047542877653 |
| }, |
| "model.language_model.layers.49.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.025304526090621948, |
| "expected_noise": 0.026052218516838404 |
| }, |
| "model.language_model.layers.49.linear_attn.in_proj_z.weight": { |
| "residual": 0.015686241909861565, |
| "expected_noise": 0.015911428515066064 |
| }, |
| "model.language_model.layers.49.linear_attn.out_proj.weight": { |
| "residual": 0.03031422570347786, |
| "expected_noise": 0.031612105944887864 |
| }, |
| "model.language_model.layers.49.mlp.down_proj.weight": { |
| "residual": 0.025021035224199295, |
| "expected_noise": 0.025654967251806274 |
| }, |
| "model.language_model.layers.49.mlp.gate_proj.weight": { |
| "residual": 0.015496470965445042, |
| "expected_noise": 0.015640358108795518 |
| }, |
| "model.language_model.layers.49.mlp.up_proj.weight": { |
| "residual": 0.01949373260140419, |
| "expected_noise": 0.01984401883705398 |
| }, |
| "model.language_model.layers.50.linear_attn.in_proj_a.weight": { |
| "residual": 0.0021821889095008373, |
| "expected_noise": 0.013036955872006708 |
| }, |
| "model.language_model.layers.50.linear_attn.in_proj_b.weight": { |
| "residual": 0.002378121018409729, |
| "expected_noise": 0.009365670649256705 |
| }, |
| "model.language_model.layers.50.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02733590453863144, |
| "expected_noise": 0.02821943031705558 |
| }, |
| "model.language_model.layers.50.linear_attn.in_proj_z.weight": { |
| "residual": 0.01663283444941044, |
| "expected_noise": 0.017077369597350674 |
| }, |
| "model.language_model.layers.50.linear_attn.out_proj.weight": { |
| "residual": 0.029521115124225616, |
| "expected_noise": 0.030780933934593614 |
| }, |
| "model.language_model.layers.50.mlp.down_proj.weight": { |
| "residual": 0.029773134738206863, |
| "expected_noise": 0.030606779908207798 |
| }, |
| "model.language_model.layers.50.mlp.gate_proj.weight": { |
| "residual": 0.015515661798417568, |
| "expected_noise": 0.015652775204659516 |
| }, |
| "model.language_model.layers.50.mlp.up_proj.weight": { |
| "residual": 0.020437853410840034, |
| "expected_noise": 0.020848004617932802 |
| }, |
| "model.language_model.layers.51.mlp.down_proj.weight": { |
| "residual": 0.030283628031611443, |
| "expected_noise": 0.031150151470709958 |
| }, |
| "model.language_model.layers.51.mlp.gate_proj.weight": { |
| "residual": 0.01711272820830345, |
| "expected_noise": 0.017347958296703915 |
| }, |
| "model.language_model.layers.51.mlp.up_proj.weight": { |
| "residual": 0.02220666967332363, |
| "expected_noise": 0.02269936063388528 |
| }, |
| "model.language_model.layers.51.self_attn.k_proj.weight": { |
| "residual": 0.02041572891175747, |
| "expected_noise": 0.02402466903633401 |
| }, |
| "model.language_model.layers.51.self_attn.o_proj.weight": { |
| "residual": 0.03522057458758354, |
| "expected_noise": 0.036799995204545854 |
| }, |
| "model.language_model.layers.51.self_attn.q_proj.weight": { |
| "residual": 0.020862102508544922, |
| "expected_noise": 0.021358853970957343 |
| }, |
| "model.language_model.layers.51.self_attn.v_proj.weight": { |
| "residual": 0.03362685441970825, |
| "expected_noise": 0.03972858625902234 |
| }, |
| "model.language_model.layers.52.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023287213407456875, |
| "expected_noise": 0.011369668532506825 |
| }, |
| "model.language_model.layers.52.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022252900525927544, |
| "expected_noise": 0.010877067016972813 |
| }, |
| "model.language_model.layers.52.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.027766192331910133, |
| "expected_noise": 0.02865892706135895 |
| }, |
| "model.language_model.layers.52.linear_attn.in_proj_z.weight": { |
| "residual": 0.018600407987833023, |
| "expected_noise": 0.019187290968053494 |
| }, |
| "model.language_model.layers.52.linear_attn.out_proj.weight": { |
| "residual": 0.030959295108914375, |
| "expected_noise": 0.03227856415833599 |
| }, |
| "model.language_model.layers.52.mlp.down_proj.weight": { |
| "residual": 0.03028861992061138, |
| "expected_noise": 0.031146630680948566 |
| }, |
| "model.language_model.layers.52.mlp.gate_proj.weight": { |
| "residual": 0.018076004460453987, |
| "expected_noise": 0.018347449738315962 |
| }, |
| "model.language_model.layers.52.mlp.up_proj.weight": { |
| "residual": 0.02267509326338768, |
| "expected_noise": 0.023178694842883776 |
| }, |
| "model.language_model.layers.53.linear_attn.in_proj_a.weight": { |
| "residual": 0.0024176444858312607, |
| "expected_noise": 0.02468933018045346 |
| }, |
| "model.language_model.layers.53.linear_attn.in_proj_b.weight": { |
| "residual": 0.002354005351662636, |
| "expected_noise": 0.008768547257193289 |
| }, |
| "model.language_model.layers.53.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02857859618961811, |
| "expected_noise": 0.0295193629450065 |
| }, |
| "model.language_model.layers.53.linear_attn.in_proj_z.weight": { |
| "residual": 0.017500586807727814, |
| "expected_noise": 0.018016796393040442 |
| }, |
| "model.language_model.layers.53.linear_attn.out_proj.weight": { |
| "residual": 0.03068842552602291, |
| "expected_noise": 0.03200879168078295 |
| }, |
| "model.language_model.layers.53.mlp.down_proj.weight": { |
| "residual": 0.030777521431446075, |
| "expected_noise": 0.03165925668055226 |
| }, |
| "model.language_model.layers.53.mlp.gate_proj.weight": { |
| "residual": 0.018804004415869713, |
| "expected_noise": 0.019129643312612035 |
| }, |
| "model.language_model.layers.53.mlp.up_proj.weight": { |
| "residual": 0.02249934710562229, |
| "expected_noise": 0.02300204059158934 |
| }, |
| "model.language_model.layers.54.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023231590166687965, |
| "expected_noise": 0.009714623226527792 |
| }, |
| "model.language_model.layers.54.linear_attn.in_proj_b.weight": { |
| "residual": 0.002327405381947756, |
| "expected_noise": 0.013712555179649017 |
| }, |
| "model.language_model.layers.54.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02942892536520958, |
| "expected_noise": 0.030409317441862253 |
| }, |
| "model.language_model.layers.54.linear_attn.in_proj_z.weight": { |
| "residual": 0.016573505476117134, |
| "expected_noise": 0.017013871604004104 |
| }, |
| "model.language_model.layers.54.linear_attn.out_proj.weight": { |
| "residual": 0.030619315803050995, |
| "expected_noise": 0.03193454240577491 |
| }, |
| "model.language_model.layers.54.mlp.down_proj.weight": { |
| "residual": 0.030573459342122078, |
| "expected_noise": 0.0314488887750836 |
| }, |
| "model.language_model.layers.54.mlp.gate_proj.weight": { |
| "residual": 0.019209347665309906, |
| "expected_noise": 0.019444042963066527 |
| }, |
| "model.language_model.layers.54.mlp.up_proj.weight": { |
| "residual": 0.02303885668516159, |
| "expected_noise": 0.023571035738158153 |
| }, |
| "model.language_model.layers.55.mlp.down_proj.weight": { |
| "residual": 0.03249518200755119, |
| "expected_noise": 0.03344270268507403 |
| }, |
| "model.language_model.layers.55.mlp.gate_proj.weight": { |
| "residual": 0.0178027655929327, |
| "expected_noise": 0.018066885605868357 |
| }, |
| "model.language_model.layers.55.mlp.up_proj.weight": { |
| "residual": 0.02315850369632244, |
| "expected_noise": 0.023694242875895364 |
| }, |
| "model.language_model.layers.55.self_attn.k_proj.weight": { |
| "residual": 0.020712995901703835, |
| "expected_noise": 0.02435315815690463 |
| }, |
| "model.language_model.layers.55.self_attn.o_proj.weight": { |
| "residual": 0.035651884973049164, |
| "expected_noise": 0.03725802423886937 |
| }, |
| "model.language_model.layers.55.self_attn.q_proj.weight": { |
| "residual": 0.013427541591227055, |
| "expected_noise": 0.013494015441156483 |
| }, |
| "model.language_model.layers.55.self_attn.v_proj.weight": { |
| "residual": 0.035547833889722824, |
| "expected_noise": 0.04200791033248108 |
| }, |
| "model.language_model.layers.56.linear_attn.in_proj_a.weight": { |
| "residual": 0.002530657686293125, |
| "expected_noise": 0.007781343141565281 |
| }, |
| "model.language_model.layers.56.linear_attn.in_proj_b.weight": { |
| "residual": 0.00227510672993958, |
| "expected_noise": 0.012104280518564683 |
| }, |
| "model.language_model.layers.56.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.028010308742523193, |
| "expected_noise": 0.028892692815948647 |
| }, |
| "model.language_model.layers.56.linear_attn.in_proj_z.weight": { |
| "residual": 0.017876602709293365, |
| "expected_noise": 0.018420378918692058 |
| }, |
| "model.language_model.layers.56.linear_attn.out_proj.weight": { |
| "residual": 0.027651440352201462, |
| "expected_noise": 0.02880636595010057 |
| }, |
| "model.language_model.layers.56.mlp.down_proj.weight": { |
| "residual": 0.03365470841526985, |
| "expected_noise": 0.03464898304237418 |
| }, |
| "model.language_model.layers.56.mlp.gate_proj.weight": { |
| "residual": 0.018467919901013374, |
| "expected_noise": 0.018767661003586032 |
| }, |
| "model.language_model.layers.56.mlp.up_proj.weight": { |
| "residual": 0.023861633613705635, |
| "expected_noise": 0.024432684052184475 |
| }, |
| "model.language_model.layers.57.linear_attn.in_proj_a.weight": { |
| "residual": 0.002384716644883156, |
| "expected_noise": 0.013886062274079998 |
| }, |
| "model.language_model.layers.57.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024188936222344637, |
| "expected_noise": 0.009796283365176 |
| }, |
| "model.language_model.layers.57.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02720802277326584, |
| "expected_noise": 0.028085018626222806 |
| }, |
| "model.language_model.layers.57.linear_attn.in_proj_z.weight": { |
| "residual": 0.02044970542192459, |
| "expected_noise": 0.02114602269464907 |
| }, |
| "model.language_model.layers.57.linear_attn.out_proj.weight": { |
| "residual": 0.03180456906557083, |
| "expected_noise": 0.03319159066755862 |
| }, |
| "model.language_model.layers.57.mlp.down_proj.weight": { |
| "residual": 0.0318509005010128, |
| "expected_noise": 0.032779519642657 |
| }, |
| "model.language_model.layers.57.mlp.gate_proj.weight": { |
| "residual": 0.020891694352030754, |
| "expected_noise": 0.02132240231663702 |
| }, |
| "model.language_model.layers.57.mlp.up_proj.weight": { |
| "residual": 0.024588799104094505, |
| "expected_noise": 0.025197557071712683 |
| }, |
| "model.language_model.layers.58.linear_attn.in_proj_a.weight": { |
| "residual": 0.0026006177067756653, |
| "expected_noise": 0.015638709613375937 |
| }, |
| "model.language_model.layers.58.linear_attn.in_proj_b.weight": { |
| "residual": 0.0025621044915169477, |
| "expected_noise": 0.007728981738109636 |
| }, |
| "model.language_model.layers.58.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.024005386978387833, |
| "expected_noise": 0.024721428501996167 |
| }, |
| "model.language_model.layers.58.linear_attn.in_proj_z.weight": { |
| "residual": 0.01443715114146471, |
| "expected_noise": 0.014656060344980227 |
| }, |
| "model.language_model.layers.58.linear_attn.out_proj.weight": { |
| "residual": 0.029232235625386238, |
| "expected_noise": 0.030477361338643684 |
| }, |
| "model.language_model.layers.58.mlp.down_proj.weight": { |
| "residual": 0.03281804546713829, |
| "expected_noise": 0.03378947637676625 |
| }, |
| "model.language_model.layers.58.mlp.gate_proj.weight": { |
| "residual": 0.02148330584168434, |
| "expected_noise": 0.021939104971958022 |
| }, |
| "model.language_model.layers.58.mlp.up_proj.weight": { |
| "residual": 0.02598547749221325, |
| "expected_noise": 0.02665407624747682 |
| }, |
| "model.language_model.layers.59.mlp.down_proj.weight": { |
| "residual": 0.035459425300359726, |
| "expected_noise": 0.036542247850026674 |
| }, |
| "model.language_model.layers.59.mlp.gate_proj.weight": { |
| "residual": 0.016634389758110046, |
| "expected_noise": 0.016845524654312567 |
| }, |
| "model.language_model.layers.59.mlp.up_proj.weight": { |
| "residual": 0.026278745383024216, |
| "expected_noise": 0.026962135694847587 |
| }, |
| "model.language_model.layers.59.self_attn.k_proj.weight": { |
| "residual": 0.0184831153601408, |
| "expected_noise": 0.021638267701486906 |
| }, |
| "model.language_model.layers.59.self_attn.o_proj.weight": { |
| "residual": 0.026731420308351517, |
| "expected_noise": 0.027832027029931677 |
| }, |
| "model.language_model.layers.59.self_attn.q_proj.weight": { |
| "residual": 0.014739111997187138, |
| "expected_noise": 0.014777830961458866 |
| }, |
| "model.language_model.layers.59.self_attn.v_proj.weight": { |
| "residual": 0.03865283355116844, |
| "expected_noise": 0.045732647940111344 |
| }, |
| "model.language_model.layers.60.linear_attn.in_proj_a.weight": { |
| "residual": 0.002289086813107133, |
| "expected_noise": 0.008267083011287903 |
| }, |
| "model.language_model.layers.60.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023570815101265907, |
| "expected_noise": 0.013177346239197592 |
| }, |
| "model.language_model.layers.60.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.024331018328666687, |
| "expected_noise": 0.025070741525026752 |
| }, |
| "model.language_model.layers.60.linear_attn.in_proj_z.weight": { |
| "residual": 0.019167112186551094, |
| "expected_noise": 0.019802231508598105 |
| }, |
| "model.language_model.layers.60.linear_attn.out_proj.weight": { |
| "residual": 0.029233647510409355, |
| "expected_noise": 0.03047890529436367 |
| }, |
| "model.language_model.layers.60.mlp.down_proj.weight": { |
| "residual": 0.03952713683247566, |
| "expected_noise": 0.04075664363902573 |
| }, |
| "model.language_model.layers.60.mlp.gate_proj.weight": { |
| "residual": 0.016480302438139915, |
| "expected_noise": 0.016687711420405416 |
| }, |
| "model.language_model.layers.60.mlp.up_proj.weight": { |
| "residual": 0.024740979075431824, |
| "expected_noise": 0.02534933391375553 |
| }, |
| "model.language_model.layers.61.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023766006343066692, |
| "expected_noise": 0.011536331254929892 |
| }, |
| "model.language_model.layers.61.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022966056130826473, |
| "expected_noise": 0.011429781610576624 |
| }, |
| "model.language_model.layers.61.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.027169818058609962, |
| "expected_noise": 0.02804815815493431 |
| }, |
| "model.language_model.layers.61.linear_attn.in_proj_z.weight": { |
| "residual": 0.017240898683667183, |
| "expected_noise": 0.017737044777722477 |
| }, |
| "model.language_model.layers.61.linear_attn.out_proj.weight": { |
| "residual": 0.029676228761672974, |
| "expected_noise": 0.030942040846998805 |
| }, |
| "model.language_model.layers.61.mlp.down_proj.weight": { |
| "residual": 0.021592097356915474, |
| "expected_noise": 0.022069107818583223 |
| }, |
| "model.language_model.layers.61.mlp.gate_proj.weight": { |
| "residual": 0.013503847643733025, |
| "expected_noise": 0.013481994916353091 |
| }, |
| "model.language_model.layers.61.mlp.up_proj.weight": { |
| "residual": 0.023468300700187683, |
| "expected_noise": 0.024022616483865392 |
| }, |
| "model.language_model.layers.62.linear_attn.in_proj_a.weight": { |
| "residual": 0.0021686514373868704, |
| "expected_noise": 0.008354297350108912 |
| }, |
| "model.language_model.layers.62.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024884738959372044, |
| "expected_noise": 0.01107918738909591 |
| }, |
| "model.language_model.layers.62.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.022786706686019897, |
| "expected_noise": 0.023435400012500853 |
| }, |
| "model.language_model.layers.62.linear_attn.in_proj_z.weight": { |
| "residual": 0.012374583631753922, |
| "expected_noise": 0.012403630877112777 |
| }, |
| "model.language_model.layers.62.linear_attn.out_proj.weight": { |
| "residual": 0.02738131210207939, |
| "expected_noise": 0.02848647881215791 |
| }, |
| "model.language_model.layers.62.mlp.down_proj.weight": { |
| "residual": 0.03585934638977051, |
| "expected_noise": 0.03695335865297991 |
| }, |
| "model.language_model.layers.62.mlp.gate_proj.weight": { |
| "residual": 0.013430722057819366, |
| "expected_noise": 0.013442489808872294 |
| }, |
| "model.language_model.layers.62.mlp.up_proj.weight": { |
| "residual": 0.017603307962417603, |
| "expected_noise": 0.017874833912440644 |
| }, |
| "model.language_model.layers.63.mlp.down_proj.weight": { |
| "residual": 0.029827678576111794, |
| "expected_noise": 0.030692818056403056 |
| }, |
| "model.language_model.layers.63.mlp.gate_proj.weight": { |
| "residual": 0.013950350694358349, |
| "expected_noise": 0.013806140324740061 |
| }, |
| "model.language_model.layers.63.mlp.up_proj.weight": { |
| "residual": 0.015037868171930313, |
| "expected_noise": 0.014954599992942602 |
| }, |
| "model.language_model.layers.63.self_attn.k_proj.weight": { |
| "residual": 0.018407564610242844, |
| "expected_noise": 0.021579426665854204 |
| }, |
| "model.language_model.layers.63.self_attn.o_proj.weight": { |
| "residual": 0.030633052811026573, |
| "expected_noise": 0.03189698165009054 |
| }, |
| "model.language_model.layers.63.self_attn.q_proj.weight": { |
| "residual": 0.013624117709696293, |
| "expected_noise": 0.01369562709404077 |
| }, |
| "model.language_model.layers.63.self_attn.v_proj.weight": { |
| "residual": 0.027015727013349533, |
| "expected_noise": 0.031955010839756416 |
| } |
| }, |
| "full_deltas": [] |
| } |
