{ "rank": 256, "max_residual": 0.05, "residuals": { "model.language_model.layers.40.linear_attn.in_proj_a.weight": { "residual": 0.002146052196621895, "expected_noise": 0.012974443843666715 }, "model.language_model.layers.40.linear_attn.in_proj_b.weight": { "residual": 0.0024941368028521538, "expected_noise": 0.012081978087675636 }, "model.language_model.layers.40.linear_attn.in_proj_qkv.weight": { "residual": 0.01444031111896038, "expected_noise": 0.014606946275233464 }, "model.language_model.layers.40.linear_attn.in_proj_z.weight": { "residual": 0.011644049547612667, "expected_noise": 0.011715060297014478 }, "model.language_model.layers.40.linear_attn.out_proj.weight": { "residual": 0.023646950721740723, "expected_noise": 0.02451279331265909 }, "model.language_model.layers.40.mlp.down_proj.weight": { "residual": 0.025049719959497452, "expected_noise": 0.02567986889698744 }, "model.language_model.layers.40.mlp.gate_proj.weight": { "residual": 0.011802067048847675, "expected_noise": 0.011712125321091709 }, "model.language_model.layers.40.mlp.up_proj.weight": { "residual": 0.018294109031558037, "expected_noise": 0.018593062788491467 }, "model.language_model.layers.41.linear_attn.in_proj_a.weight": { "residual": 0.0023987703025341034, "expected_noise": 0.015031720988324622 }, "model.language_model.layers.41.linear_attn.in_proj_b.weight": { "residual": 0.002487862017005682, "expected_noise": 0.010877011116313404 }, "model.language_model.layers.41.linear_attn.in_proj_qkv.weight": { "residual": 0.016599491238594055, "expected_noise": 0.01681164873421441 }, "model.language_model.layers.41.linear_attn.in_proj_z.weight": { "residual": 0.015272041782736778, "expected_noise": 0.01562853986274499 }, "model.language_model.layers.41.linear_attn.out_proj.weight": { "residual": 0.02833371050655842, "expected_noise": 0.029499450273880207 }, "model.language_model.layers.41.mlp.down_proj.weight": { "residual": 0.029693936929106712, "expected_noise": 0.030523855890532622 }, "model.language_model.layers.41.mlp.gate_proj.weight": { "residual": 0.014496462419629097, "expected_noise": 0.0145778493576875 }, "model.language_model.layers.41.mlp.up_proj.weight": { "residual": 0.019872894510626793, "expected_noise": 0.020249397125064576 }, "model.language_model.layers.42.linear_attn.in_proj_a.weight": { "residual": 0.002244287636131048, "expected_noise": 0.012608820256705932 }, "model.language_model.layers.42.linear_attn.in_proj_b.weight": { "residual": 0.0024181529879570007, "expected_noise": 0.009967008221082102 }, "model.language_model.layers.42.linear_attn.in_proj_qkv.weight": { "residual": 0.020915284752845764, "expected_noise": 0.02146663214507366 }, "model.language_model.layers.42.linear_attn.in_proj_z.weight": { "residual": 0.014570524916052818, "expected_noise": 0.014861297978906701 }, "model.language_model.layers.42.linear_attn.out_proj.weight": { "residual": 0.030799880623817444, "expected_noise": 0.03211886467913097 }, "model.language_model.layers.42.mlp.down_proj.weight": { "residual": 0.030454199761152267, "expected_noise": 0.031328033541941205 }, "model.language_model.layers.42.mlp.gate_proj.weight": { "residual": 0.016223207116127014, "expected_noise": 0.01640663208177938 }, "model.language_model.layers.42.mlp.up_proj.weight": { "residual": 0.020855600014328957, "expected_noise": 0.02127874308905236 }, "model.language_model.layers.43.mlp.down_proj.weight": { "residual": 0.032823145389556885, "expected_noise": 0.03378377377276178 }, "model.language_model.layers.43.mlp.gate_proj.weight": { "residual": 0.019350314512848854, "expected_noise": 0.01969872596677415 }, "model.language_model.layers.43.mlp.up_proj.weight": { "residual": 0.02309340238571167, "expected_noise": 0.023623713819061757 }, "model.language_model.layers.43.self_attn.k_proj.weight": { "residual": 0.02066306583583355, "expected_noise": 0.02434440532880551 }, "model.language_model.layers.43.self_attn.o_proj.weight": { "residual": 0.03257371485233307, "expected_noise": 0.033977171990374706 }, "model.language_model.layers.43.self_attn.q_proj.weight": { "residual": 0.02321186102926731, "expected_noise": 0.02383428591468695 }, "model.language_model.layers.43.self_attn.v_proj.weight": { "residual": 0.02583351917564869, "expected_noise": 0.030454820453565307 }, "model.language_model.layers.44.linear_attn.in_proj_a.weight": { "residual": 0.002216341206803918, "expected_noise": 0.012409744532101868 }, "model.language_model.layers.44.linear_attn.in_proj_b.weight": { "residual": 0.002490745158866048, "expected_noise": 0.009856438750009123 }, "model.language_model.layers.44.linear_attn.in_proj_qkv.weight": { "residual": 0.025673801079392433, "expected_noise": 0.02647559578788941 }, "model.language_model.layers.44.linear_attn.in_proj_z.weight": { "residual": 0.017309727147221565, "expected_noise": 0.0177499164476529 }, "model.language_model.layers.44.linear_attn.out_proj.weight": { "residual": 0.03230440989136696, "expected_noise": 0.033712908833213345 }, "model.language_model.layers.44.mlp.down_proj.weight": { "residual": 0.031004754826426506, "expected_noise": 0.03189631559201829 }, "model.language_model.layers.44.mlp.gate_proj.weight": { "residual": 0.01922808773815632, "expected_noise": 0.01957301177624235 }, "model.language_model.layers.44.mlp.up_proj.weight": { "residual": 0.022256333380937576, "expected_noise": 0.022746449441290693 }, "model.language_model.layers.45.linear_attn.in_proj_a.weight": { "residual": 0.0022388342767953873, "expected_noise": 0.014264842383568442 }, "model.language_model.layers.45.linear_attn.in_proj_b.weight": { "residual": 0.002298386301845312, "expected_noise": 0.010848777437960757 }, "model.language_model.layers.45.linear_attn.in_proj_qkv.weight": { "residual": 0.024941040202975273, "expected_noise": 0.02570060683459132 }, "model.language_model.layers.45.linear_attn.in_proj_z.weight": { "residual": 0.01844269037246704, "expected_noise": 0.019001911225722905 }, "model.language_model.layers.45.linear_attn.out_proj.weight": { "residual": 0.029758809134364128, "expected_noise": 0.031028814902908496 }, "model.language_model.layers.45.mlp.down_proj.weight": { "residual": 0.02979564666748047, "expected_noise": 0.0306394651651357 }, "model.language_model.layers.45.mlp.gate_proj.weight": { "residual": 0.018665647134184837, "expected_noise": 0.0189769975480935 }, "model.language_model.layers.45.mlp.up_proj.weight": { "residual": 0.02305259369313717, "expected_noise": 0.02357369985563593 }, "model.language_model.layers.46.linear_attn.in_proj_a.weight": { "residual": 0.0021862995345145464, "expected_noise": 0.0136549115320459 }, "model.language_model.layers.46.linear_attn.in_proj_b.weight": { "residual": 0.002302567008882761, "expected_noise": 0.011588226213113176 }, "model.language_model.layers.46.linear_attn.in_proj_qkv.weight": { "residual": 0.02518370747566223, "expected_noise": 0.02595827159156582 }, "model.language_model.layers.46.linear_attn.in_proj_z.weight": { "residual": 0.016484098508954048, "expected_noise": 0.01678300866186771 }, "model.language_model.layers.46.linear_attn.out_proj.weight": { "residual": 0.030089348554611206, "expected_noise": 0.03137046317394857 }, "model.language_model.layers.46.mlp.down_proj.weight": { "residual": 0.02790791355073452, "expected_noise": 0.0286551141854796 }, "model.language_model.layers.46.mlp.gate_proj.weight": { "residual": 0.018145939335227013, "expected_noise": 0.018439166107186968 }, "model.language_model.layers.46.mlp.up_proj.weight": { "residual": 0.02233639545738697, "expected_noise": 0.022834483731352197 }, "model.language_model.layers.47.mlp.down_proj.weight": { "residual": 0.023689016699790955, "expected_noise": 0.024249328031035355 }, "model.language_model.layers.47.mlp.gate_proj.weight": { "residual": 0.016771014779806137, "expected_noise": 0.01696912237101654 }, "model.language_model.layers.47.mlp.up_proj.weight": { "residual": 0.02183159440755844, "expected_noise": 0.022308856835644132 }, "model.language_model.layers.47.self_attn.k_proj.weight": { "residual": 0.021325351670384407, "expected_noise": 0.025119833629998575 }, "model.language_model.layers.47.self_attn.o_proj.weight": { "residual": 0.032491233199834824, "expected_noise": 0.03391210515796512 }, "model.language_model.layers.47.self_attn.q_proj.weight": { "residual": 0.023718250915408134, "expected_noise": 0.024367478366397438 }, "model.language_model.layers.47.self_attn.v_proj.weight": { "residual": 0.033579420298337936, "expected_noise": 0.03972490845880818 }, "model.language_model.layers.48.linear_attn.in_proj_a.weight": { "residual": 0.002346278168261051, "expected_noise": 0.011280545329459832 }, "model.language_model.layers.48.linear_attn.in_proj_b.weight": { "residual": 0.0022730999626219273, "expected_noise": 0.009259874055325875 }, "model.language_model.layers.48.linear_attn.in_proj_qkv.weight": { "residual": 0.025525910779833794, "expected_noise": 0.026318426907857614 }, "model.language_model.layers.48.linear_attn.in_proj_z.weight": { "residual": 0.014913615770637989, "expected_noise": 0.01523274634308516 }, "model.language_model.layers.48.linear_attn.out_proj.weight": { "residual": 0.025087514892220497, "expected_noise": 0.0260888624832962 }, "model.language_model.layers.48.mlp.down_proj.weight": { "residual": 0.02170288935303688, "expected_noise": 0.02215326645805858 }, "model.language_model.layers.48.mlp.gate_proj.weight": { "residual": 0.014320943504571915, "expected_noise": 0.0143976298961414 }, "model.language_model.layers.48.mlp.up_proj.weight": { "residual": 0.02098502404987812, "expected_noise": 0.021421334949477722 }, "model.language_model.layers.49.linear_attn.in_proj_a.weight": { "residual": 0.0024136558640748262, "expected_noise": 0.018704550434213242 }, "model.language_model.layers.49.linear_attn.in_proj_b.weight": { "residual": 0.0024829739704728127, "expected_noise": 0.010717047542877653 }, "model.language_model.layers.49.linear_attn.in_proj_qkv.weight": { "residual": 0.025304526090621948, "expected_noise": 0.026052218516838404 }, "model.language_model.layers.49.linear_attn.in_proj_z.weight": { "residual": 0.015686241909861565, "expected_noise": 0.015911428515066064 }, "model.language_model.layers.49.linear_attn.out_proj.weight": { "residual": 0.03031422570347786, "expected_noise": 0.031612105944887864 }, "model.language_model.layers.49.mlp.down_proj.weight": { "residual": 0.025021035224199295, "expected_noise": 0.025654967251806274 }, "model.language_model.layers.49.mlp.gate_proj.weight": { "residual": 0.015496470965445042, "expected_noise": 0.015640358108795518 }, "model.language_model.layers.49.mlp.up_proj.weight": { "residual": 0.01949373260140419, "expected_noise": 0.01984401883705398 }, "model.language_model.layers.50.linear_attn.in_proj_a.weight": { "residual": 0.0021821889095008373, "expected_noise": 0.013036955872006708 }, "model.language_model.layers.50.linear_attn.in_proj_b.weight": { "residual": 0.002378121018409729, "expected_noise": 0.009365670649256705 }, "model.language_model.layers.50.linear_attn.in_proj_qkv.weight": { "residual": 0.02733590453863144, "expected_noise": 0.02821943031705558 }, "model.language_model.layers.50.linear_attn.in_proj_z.weight": { "residual": 0.01663283444941044, "expected_noise": 0.017077369597350674 }, "model.language_model.layers.50.linear_attn.out_proj.weight": { "residual": 0.029521115124225616, "expected_noise": 0.030780933934593614 }, "model.language_model.layers.50.mlp.down_proj.weight": { "residual": 0.029773134738206863, "expected_noise": 0.030606779908207798 }, "model.language_model.layers.50.mlp.gate_proj.weight": { "residual": 0.015515661798417568, "expected_noise": 0.015652775204659516 }, "model.language_model.layers.50.mlp.up_proj.weight": { "residual": 0.020437853410840034, "expected_noise": 0.020848004617932802 }, "model.language_model.layers.51.mlp.down_proj.weight": { "residual": 0.030283628031611443, "expected_noise": 0.031150151470709958 }, "model.language_model.layers.51.mlp.gate_proj.weight": { "residual": 0.01711272820830345, "expected_noise": 0.017347958296703915 }, "model.language_model.layers.51.mlp.up_proj.weight": { "residual": 0.02220666967332363, "expected_noise": 0.02269936063388528 }, "model.language_model.layers.51.self_attn.k_proj.weight": { "residual": 0.02041572891175747, "expected_noise": 0.02402466903633401 }, "model.language_model.layers.51.self_attn.o_proj.weight": { "residual": 0.03522057458758354, "expected_noise": 0.036799995204545854 }, "model.language_model.layers.51.self_attn.q_proj.weight": { "residual": 0.020862102508544922, "expected_noise": 0.021358853970957343 }, "model.language_model.layers.51.self_attn.v_proj.weight": { "residual": 0.03362685441970825, "expected_noise": 0.03972858625902234 }, "model.language_model.layers.52.linear_attn.in_proj_a.weight": { "residual": 0.0023287213407456875, "expected_noise": 0.011369668532506825 }, "model.language_model.layers.52.linear_attn.in_proj_b.weight": { "residual": 0.0022252900525927544, "expected_noise": 0.010877067016972813 }, "model.language_model.layers.52.linear_attn.in_proj_qkv.weight": { "residual": 0.027766192331910133, "expected_noise": 0.02865892706135895 }, "model.language_model.layers.52.linear_attn.in_proj_z.weight": { "residual": 0.018600407987833023, "expected_noise": 0.019187290968053494 }, "model.language_model.layers.52.linear_attn.out_proj.weight": { "residual": 0.030959295108914375, "expected_noise": 0.03227856415833599 }, "model.language_model.layers.52.mlp.down_proj.weight": { "residual": 0.03028861992061138, "expected_noise": 0.031146630680948566 }, "model.language_model.layers.52.mlp.gate_proj.weight": { "residual": 0.018076004460453987, "expected_noise": 0.018347449738315962 }, "model.language_model.layers.52.mlp.up_proj.weight": { "residual": 0.02267509326338768, "expected_noise": 0.023178694842883776 }, "model.language_model.layers.53.linear_attn.in_proj_a.weight": { "residual": 0.0024176444858312607, "expected_noise": 0.02468933018045346 }, "model.language_model.layers.53.linear_attn.in_proj_b.weight": { "residual": 0.002354005351662636, "expected_noise": 0.008768547257193289 }, "model.language_model.layers.53.linear_attn.in_proj_qkv.weight": { "residual": 0.02857859618961811, "expected_noise": 0.0295193629450065 }, "model.language_model.layers.53.linear_attn.in_proj_z.weight": { "residual": 0.017500586807727814, "expected_noise": 0.018016796393040442 }, "model.language_model.layers.53.linear_attn.out_proj.weight": { "residual": 0.03068842552602291, "expected_noise": 0.03200879168078295 }, "model.language_model.layers.53.mlp.down_proj.weight": { "residual": 0.030777521431446075, "expected_noise": 0.03165925668055226 }, "model.language_model.layers.53.mlp.gate_proj.weight": { "residual": 0.018804004415869713, "expected_noise": 0.019129643312612035 }, "model.language_model.layers.53.mlp.up_proj.weight": { "residual": 0.02249934710562229, "expected_noise": 0.02300204059158934 }, "model.language_model.layers.54.linear_attn.in_proj_a.weight": { "residual": 0.0023231590166687965, "expected_noise": 0.009714623226527792 }, "model.language_model.layers.54.linear_attn.in_proj_b.weight": { "residual": 0.002327405381947756, "expected_noise": 0.013712555179649017 }, "model.language_model.layers.54.linear_attn.in_proj_qkv.weight": { "residual": 0.02942892536520958, "expected_noise": 0.030409317441862253 }, "model.language_model.layers.54.linear_attn.in_proj_z.weight": { "residual": 0.016573505476117134, "expected_noise": 0.017013871604004104 }, "model.language_model.layers.54.linear_attn.out_proj.weight": { "residual": 0.030619315803050995, "expected_noise": 0.03193454240577491 }, "model.language_model.layers.54.mlp.down_proj.weight": { "residual": 0.030573459342122078, "expected_noise": 0.0314488887750836 }, "model.language_model.layers.54.mlp.gate_proj.weight": { "residual": 0.019209347665309906, "expected_noise": 0.019444042963066527 }, "model.language_model.layers.54.mlp.up_proj.weight": { "residual": 0.02303885668516159, "expected_noise": 0.023571035738158153 }, "model.language_model.layers.55.mlp.down_proj.weight": { "residual": 0.03249518200755119, "expected_noise": 0.03344270268507403 }, "model.language_model.layers.55.mlp.gate_proj.weight": { "residual": 0.0178027655929327, "expected_noise": 0.018066885605868357 }, "model.language_model.layers.55.mlp.up_proj.weight": { "residual": 0.02315850369632244, "expected_noise": 0.023694242875895364 }, "model.language_model.layers.55.self_attn.k_proj.weight": { "residual": 0.020712995901703835, "expected_noise": 0.02435315815690463 }, "model.language_model.layers.55.self_attn.o_proj.weight": { "residual": 0.035651884973049164, "expected_noise": 0.03725802423886937 }, "model.language_model.layers.55.self_attn.q_proj.weight": { "residual": 0.013427541591227055, "expected_noise": 0.013494015441156483 }, "model.language_model.layers.55.self_attn.v_proj.weight": { "residual": 0.035547833889722824, "expected_noise": 0.04200791033248108 }, "model.language_model.layers.56.linear_attn.in_proj_a.weight": { "residual": 0.002530657686293125, "expected_noise": 0.007781343141565281 }, "model.language_model.layers.56.linear_attn.in_proj_b.weight": { "residual": 0.00227510672993958, "expected_noise": 0.012104280518564683 }, "model.language_model.layers.56.linear_attn.in_proj_qkv.weight": { "residual": 0.028010308742523193, "expected_noise": 0.028892692815948647 }, "model.language_model.layers.56.linear_attn.in_proj_z.weight": { "residual": 0.017876602709293365, "expected_noise": 0.018420378918692058 }, "model.language_model.layers.56.linear_attn.out_proj.weight": { "residual": 0.027651440352201462, "expected_noise": 0.02880636595010057 }, "model.language_model.layers.56.mlp.down_proj.weight": { "residual": 0.03365470841526985, "expected_noise": 0.03464898304237418 }, "model.language_model.layers.56.mlp.gate_proj.weight": { "residual": 0.018467919901013374, "expected_noise": 0.018767661003586032 }, "model.language_model.layers.56.mlp.up_proj.weight": { "residual": 0.023861633613705635, "expected_noise": 0.024432684052184475 }, "model.language_model.layers.57.linear_attn.in_proj_a.weight": { "residual": 0.002384716644883156, "expected_noise": 0.013886062274079998 }, "model.language_model.layers.57.linear_attn.in_proj_b.weight": { "residual": 0.0024188936222344637, "expected_noise": 0.009796283365176 }, "model.language_model.layers.57.linear_attn.in_proj_qkv.weight": { "residual": 0.02720802277326584, "expected_noise": 0.028085018626222806 }, "model.language_model.layers.57.linear_attn.in_proj_z.weight": { "residual": 0.02044970542192459, "expected_noise": 0.02114602269464907 }, "model.language_model.layers.57.linear_attn.out_proj.weight": { "residual": 0.03180456906557083, "expected_noise": 0.03319159066755862 }, "model.language_model.layers.57.mlp.down_proj.weight": { "residual": 0.0318509005010128, "expected_noise": 0.032779519642657 }, "model.language_model.layers.57.mlp.gate_proj.weight": { "residual": 0.020891694352030754, "expected_noise": 0.02132240231663702 }, "model.language_model.layers.57.mlp.up_proj.weight": { "residual": 0.024588799104094505, "expected_noise": 0.025197557071712683 }, "model.language_model.layers.58.linear_attn.in_proj_a.weight": { "residual": 0.0026006177067756653, "expected_noise": 0.015638709613375937 }, "model.language_model.layers.58.linear_attn.in_proj_b.weight": { "residual": 0.0025621044915169477, "expected_noise": 0.007728981738109636 }, "model.language_model.layers.58.linear_attn.in_proj_qkv.weight": { "residual": 0.024005386978387833, "expected_noise": 0.024721428501996167 }, "model.language_model.layers.58.linear_attn.in_proj_z.weight": { "residual": 0.01443715114146471, "expected_noise": 0.014656060344980227 }, "model.language_model.layers.58.linear_attn.out_proj.weight": { "residual": 0.029232235625386238, "expected_noise": 0.030477361338643684 }, "model.language_model.layers.58.mlp.down_proj.weight": { "residual": 0.03281804546713829, "expected_noise": 0.03378947637676625 }, "model.language_model.layers.58.mlp.gate_proj.weight": { "residual": 0.02148330584168434, "expected_noise": 0.021939104971958022 }, "model.language_model.layers.58.mlp.up_proj.weight": { "residual": 0.02598547749221325, "expected_noise": 0.02665407624747682 }, "model.language_model.layers.59.mlp.down_proj.weight": { "residual": 0.035459425300359726, "expected_noise": 0.036542247850026674 }, "model.language_model.layers.59.mlp.gate_proj.weight": { "residual": 0.016634389758110046, "expected_noise": 0.016845524654312567 }, "model.language_model.layers.59.mlp.up_proj.weight": { "residual": 0.026278745383024216, "expected_noise": 0.026962135694847587 }, "model.language_model.layers.59.self_attn.k_proj.weight": { "residual": 0.0184831153601408, "expected_noise": 0.021638267701486906 }, "model.language_model.layers.59.self_attn.o_proj.weight": { "residual": 0.026731420308351517, "expected_noise": 0.027832027029931677 }, "model.language_model.layers.59.self_attn.q_proj.weight": { "residual": 0.014739111997187138, "expected_noise": 0.014777830961458866 }, "model.language_model.layers.59.self_attn.v_proj.weight": { "residual": 0.03865283355116844, "expected_noise": 0.045732647940111344 }, "model.language_model.layers.60.linear_attn.in_proj_a.weight": { "residual": 0.002289086813107133, "expected_noise": 0.008267083011287903 }, "model.language_model.layers.60.linear_attn.in_proj_b.weight": { "residual": 0.0023570815101265907, "expected_noise": 0.013177346239197592 }, "model.language_model.layers.60.linear_attn.in_proj_qkv.weight": { "residual": 0.024331018328666687, "expected_noise": 0.025070741525026752 }, "model.language_model.layers.60.linear_attn.in_proj_z.weight": { "residual": 0.019167112186551094, "expected_noise": 0.019802231508598105 }, "model.language_model.layers.60.linear_attn.out_proj.weight": { "residual": 0.029233647510409355, "expected_noise": 0.03047890529436367 }, "model.language_model.layers.60.mlp.down_proj.weight": { "residual": 0.03952713683247566, "expected_noise": 0.04075664363902573 }, "model.language_model.layers.60.mlp.gate_proj.weight": { "residual": 0.016480302438139915, "expected_noise": 0.016687711420405416 }, "model.language_model.layers.60.mlp.up_proj.weight": { "residual": 0.024740979075431824, "expected_noise": 0.02534933391375553 }, "model.language_model.layers.61.linear_attn.in_proj_a.weight": { "residual": 0.0023766006343066692, "expected_noise": 0.011536331254929892 }, "model.language_model.layers.61.linear_attn.in_proj_b.weight": { "residual": 0.0022966056130826473, "expected_noise": 0.011429781610576624 }, "model.language_model.layers.61.linear_attn.in_proj_qkv.weight": { "residual": 0.027169818058609962, "expected_noise": 0.02804815815493431 }, "model.language_model.layers.61.linear_attn.in_proj_z.weight": { "residual": 0.017240898683667183, "expected_noise": 0.017737044777722477 }, "model.language_model.layers.61.linear_attn.out_proj.weight": { "residual": 0.029676228761672974, "expected_noise": 0.030942040846998805 }, "model.language_model.layers.61.mlp.down_proj.weight": { "residual": 0.021592097356915474, "expected_noise": 0.022069107818583223 }, "model.language_model.layers.61.mlp.gate_proj.weight": { "residual": 0.013503847643733025, "expected_noise": 0.013481994916353091 }, "model.language_model.layers.61.mlp.up_proj.weight": { "residual": 0.023468300700187683, "expected_noise": 0.024022616483865392 }, "model.language_model.layers.62.linear_attn.in_proj_a.weight": { "residual": 0.0021686514373868704, "expected_noise": 0.008354297350108912 }, "model.language_model.layers.62.linear_attn.in_proj_b.weight": { "residual": 0.0024884738959372044, "expected_noise": 0.01107918738909591 }, "model.language_model.layers.62.linear_attn.in_proj_qkv.weight": { "residual": 0.022786706686019897, "expected_noise": 0.023435400012500853 }, "model.language_model.layers.62.linear_attn.in_proj_z.weight": { "residual": 0.012374583631753922, "expected_noise": 0.012403630877112777 }, "model.language_model.layers.62.linear_attn.out_proj.weight": { "residual": 0.02738131210207939, "expected_noise": 0.02848647881215791 }, "model.language_model.layers.62.mlp.down_proj.weight": { "residual": 0.03585934638977051, "expected_noise": 0.03695335865297991 }, "model.language_model.layers.62.mlp.gate_proj.weight": { "residual": 0.013430722057819366, "expected_noise": 0.013442489808872294 }, "model.language_model.layers.62.mlp.up_proj.weight": { "residual": 0.017603307962417603, "expected_noise": 0.017874833912440644 }, "model.language_model.layers.63.mlp.down_proj.weight": { "residual": 0.029827678576111794, "expected_noise": 0.030692818056403056 }, "model.language_model.layers.63.mlp.gate_proj.weight": { "residual": 0.013950350694358349, "expected_noise": 0.013806140324740061 }, "model.language_model.layers.63.mlp.up_proj.weight": { "residual": 0.015037868171930313, "expected_noise": 0.014954599992942602 }, "model.language_model.layers.63.self_attn.k_proj.weight": { "residual": 0.018407564610242844, "expected_noise": 0.021579426665854204 }, "model.language_model.layers.63.self_attn.o_proj.weight": { "residual": 0.030633052811026573, "expected_noise": 0.03189698165009054 }, "model.language_model.layers.63.self_attn.q_proj.weight": { "residual": 0.013624117709696293, "expected_noise": 0.01369562709404077 }, "model.language_model.layers.63.self_attn.v_proj.weight": { "residual": 0.027015727013349533, "expected_noise": 0.031955010839756416 } }, "full_deltas": [] }