| { |
| "rank": 256, |
| "max_residual": 0.05, |
| "residuals": { |
| "model.language_model.layers.0.linear_attn.in_proj_a.weight": { |
| "residual": 0.002481166971847415, |
| "expected_noise": 0.02831156518707557 |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023871413432061672, |
| "expected_noise": 0.008349564530849186 |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.028010139241814613, |
| "expected_noise": 0.029223767926274215 |
| }, |
| "model.language_model.layers.0.linear_attn.in_proj_z.weight": { |
| "residual": 0.028452618047595024, |
| "expected_noise": 0.03020173782515911 |
| }, |
| "model.language_model.layers.0.linear_attn.out_proj.weight": { |
| "residual": 0.020905354991555214, |
| "expected_noise": 0.02208137960709428 |
| }, |
| "model.language_model.layers.0.mlp.down_proj.weight": { |
| "residual": 0.028096821159124374, |
| "expected_noise": 0.02914386182328274 |
| }, |
| "model.language_model.layers.0.mlp.gate_proj.weight": { |
| "residual": 0.019913610070943832, |
| "expected_noise": 0.020496658367335392 |
| }, |
| "model.language_model.layers.0.mlp.up_proj.weight": { |
| "residual": 0.018833406269550323, |
| "expected_noise": 0.019342525338178627 |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_a.weight": { |
| "residual": 0.0021766992285847664, |
| "expected_noise": 0.02457133336293675 |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023660471197217703, |
| "expected_noise": 0.025834611971101946 |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.020123932510614395, |
| "expected_noise": 0.020844547236918608 |
| }, |
| "model.language_model.layers.1.linear_attn.in_proj_z.weight": { |
| "residual": 0.017835108563303947, |
| "expected_noise": 0.018704511759167593 |
| }, |
| "model.language_model.layers.1.linear_attn.out_proj.weight": { |
| "residual": 0.027951691299676895, |
| "expected_noise": 0.029649845699456806 |
| }, |
| "model.language_model.layers.1.mlp.down_proj.weight": { |
| "residual": 0.028831562027335167, |
| "expected_noise": 0.029920130335982904 |
| }, |
| "model.language_model.layers.1.mlp.gate_proj.weight": { |
| "residual": 0.020388875156641006, |
| "expected_noise": 0.02099758040634284 |
| }, |
| "model.language_model.layers.1.mlp.up_proj.weight": { |
| "residual": 0.019373726099729538, |
| "expected_noise": 0.019923920455313913 |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_a.weight": { |
| "residual": 0.0020529604516923428, |
| "expected_noise": 0.020798647550566877 |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_b.weight": { |
| "residual": 0.002389292698353529, |
| "expected_noise": 0.015135721668016834 |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02783159725368023, |
| "expected_noise": 0.029032253214033917 |
| }, |
| "model.language_model.layers.10.linear_attn.in_proj_z.weight": { |
| "residual": 0.018996383994817734, |
| "expected_noise": 0.01997516128233796 |
| }, |
| "model.language_model.layers.10.linear_attn.out_proj.weight": { |
| "residual": 0.028022708371281624, |
| "expected_noise": 0.029723322168313013 |
| }, |
| "model.language_model.layers.10.mlp.down_proj.weight": { |
| "residual": 0.031969230622053146, |
| "expected_noise": 0.03323809081162423 |
| }, |
| "model.language_model.layers.10.mlp.gate_proj.weight": { |
| "residual": 0.019533751532435417, |
| "expected_noise": 0.02009117624690135 |
| }, |
| "model.language_model.layers.10.mlp.up_proj.weight": { |
| "residual": 0.020111756399273872, |
| "expected_noise": 0.02071079010494138 |
| }, |
| "model.language_model.layers.11.mlp.down_proj.weight": { |
| "residual": 0.031762782484292984, |
| "expected_noise": 0.033010471921525224 |
| }, |
| "model.language_model.layers.11.mlp.gate_proj.weight": { |
| "residual": 0.019560981541872025, |
| "expected_noise": 0.020123060677130513 |
| }, |
| "model.language_model.layers.11.mlp.up_proj.weight": { |
| "residual": 0.020869692787528038, |
| "expected_noise": 0.02150906591768223 |
| }, |
| "model.language_model.layers.11.self_attn.k_proj.weight": { |
| "residual": 0.022297900170087814, |
| "expected_noise": 0.02651517617405365 |
| }, |
| "model.language_model.layers.11.self_attn.o_proj.weight": { |
| "residual": 0.028615646064281464, |
| "expected_noise": 0.030356000289846086 |
| }, |
| "model.language_model.layers.11.self_attn.q_proj.weight": { |
| "residual": 0.025984665378928185, |
| "expected_noise": 0.02706973672480417 |
| }, |
| "model.language_model.layers.11.self_attn.v_proj.weight": { |
| "residual": 0.025407293811440468, |
| "expected_noise": 0.03013468773536543 |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_a.weight": { |
| "residual": 0.0024485806934535503, |
| "expected_noise": 0.019420372968141388 |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_b.weight": { |
| "residual": 0.002365445252507925, |
| "expected_noise": 0.018309427496431052 |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.028319071978330612, |
| "expected_noise": 0.029540904534968285 |
| }, |
| "model.language_model.layers.12.linear_attn.in_proj_z.weight": { |
| "residual": 0.020267268642783165, |
| "expected_noise": 0.02135990072261099 |
| }, |
| "model.language_model.layers.12.linear_attn.out_proj.weight": { |
| "residual": 0.027437621727585793, |
| "expected_noise": 0.02908597395512137 |
| }, |
| "model.language_model.layers.12.mlp.down_proj.weight": { |
| "residual": 0.029705103486776352, |
| "expected_noise": 0.030844441787333636 |
| }, |
| "model.language_model.layers.12.mlp.gate_proj.weight": { |
| "residual": 0.0185822993516922, |
| "expected_noise": 0.019082645358578235 |
| }, |
| "model.language_model.layers.12.mlp.up_proj.weight": { |
| "residual": 0.02072763442993164, |
| "expected_noise": 0.021360933630008464 |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_a.weight": { |
| "residual": 0.002337323036044836, |
| "expected_noise": 0.01972010034582101 |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_b.weight": { |
| "residual": 0.002142813056707382, |
| "expected_noise": 0.012867415629547428 |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.028196075931191444, |
| "expected_noise": 0.02941212447972328 |
| }, |
| "model.language_model.layers.13.linear_attn.in_proj_z.weight": { |
| "residual": 0.01853754185140133, |
| "expected_noise": 0.019476448038852248 |
| }, |
| "model.language_model.layers.13.linear_attn.out_proj.weight": { |
| "residual": 0.02737974189221859, |
| "expected_noise": 0.02902977297271003 |
| }, |
| "model.language_model.layers.13.mlp.down_proj.weight": { |
| "residual": 0.030016960576176643, |
| "expected_noise": 0.03117935734213052 |
| }, |
| "model.language_model.layers.13.mlp.gate_proj.weight": { |
| "residual": 0.018626024946570396, |
| "expected_noise": 0.019128662090394866 |
| }, |
| "model.language_model.layers.13.mlp.up_proj.weight": { |
| "residual": 0.02039594016969204, |
| "expected_noise": 0.021011424876891414 |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_a.weight": { |
| "residual": 0.0022698314860463142, |
| "expected_noise": 0.01820066596925822 |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_b.weight": { |
| "residual": 0.002186872297897935, |
| "expected_noise": 0.011911765813455063 |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.027325650677084923, |
| "expected_noise": 0.028500641387971117 |
| }, |
| "model.language_model.layers.14.linear_attn.in_proj_z.weight": { |
| "residual": 0.017237689346075058, |
| "expected_noise": 0.018069063693117022 |
| }, |
| "model.language_model.layers.14.linear_attn.out_proj.weight": { |
| "residual": 0.027270525693893433, |
| "expected_noise": 0.028916246566589286 |
| }, |
| "model.language_model.layers.14.mlp.down_proj.weight": { |
| "residual": 0.031754836440086365, |
| "expected_noise": 0.03300361054365933 |
| }, |
| "model.language_model.layers.14.mlp.gate_proj.weight": { |
| "residual": 0.01796301081776619, |
| "expected_noise": 0.018427800495340173 |
| }, |
| "model.language_model.layers.14.mlp.up_proj.weight": { |
| "residual": 0.02063777483999729, |
| "expected_noise": 0.02126145239538796 |
| }, |
| "model.language_model.layers.15.mlp.down_proj.weight": { |
| "residual": 0.02903105691075325, |
| "expected_noise": 0.030127538858782624 |
| }, |
| "model.language_model.layers.15.mlp.gate_proj.weight": { |
| "residual": 0.01839308999478817, |
| "expected_noise": 0.01887707116356956 |
| }, |
| "model.language_model.layers.15.mlp.up_proj.weight": { |
| "residual": 0.020844554528594017, |
| "expected_noise": 0.02148397871182622 |
| }, |
| "model.language_model.layers.15.self_attn.k_proj.weight": { |
| "residual": 0.021484728902578354, |
| "expected_noise": 0.02545731775315729 |
| }, |
| "model.language_model.layers.15.self_attn.o_proj.weight": { |
| "residual": 0.02914666198194027, |
| "expected_noise": 0.030932190911838176 |
| }, |
| "model.language_model.layers.15.self_attn.q_proj.weight": { |
| "residual": 0.02431359328329563, |
| "expected_noise": 0.025296783154912485 |
| }, |
| "model.language_model.layers.15.self_attn.v_proj.weight": { |
| "residual": 0.029649870470166206, |
| "expected_noise": 0.03523204815848145 |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_a.weight": { |
| "residual": 0.002128089312463999, |
| "expected_noise": 0.016310980441275585 |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023869434371590614, |
| "expected_noise": 0.010101458293557389 |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.028700554743409157, |
| "expected_noise": 0.02994782085014652 |
| }, |
| "model.language_model.layers.16.linear_attn.in_proj_z.weight": { |
| "residual": 0.019220247864723206, |
| "expected_noise": 0.020221943151664038 |
| }, |
| "model.language_model.layers.16.linear_attn.out_proj.weight": { |
| "residual": 0.02426838129758835, |
| "expected_noise": 0.025670437003088648 |
| }, |
| "model.language_model.layers.16.mlp.down_proj.weight": { |
| "residual": 0.022761620581150055, |
| "expected_noise": 0.023513867209505185 |
| }, |
| "model.language_model.layers.16.mlp.gate_proj.weight": { |
| "residual": 0.016623331233859062, |
| "expected_noise": 0.01699752407774793 |
| }, |
| "model.language_model.layers.16.mlp.up_proj.weight": { |
| "residual": 0.01988464966416359, |
| "expected_noise": 0.020462601790522637 |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_a.weight": { |
| "residual": 0.0024967638310045004, |
| "expected_noise": 0.018321512866938 |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022686421871185303, |
| "expected_noise": 0.014988820124913314 |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.027975285425782204, |
| "expected_noise": 0.029187326311521176 |
| }, |
| "model.language_model.layers.17.linear_attn.in_proj_z.weight": { |
| "residual": 0.014796235598623753, |
| "expected_noise": 0.015400105600467064 |
| }, |
| "model.language_model.layers.17.linear_attn.out_proj.weight": { |
| "residual": 0.024706946685910225, |
| "expected_noise": 0.026135422099318385 |
| }, |
| "model.language_model.layers.17.mlp.down_proj.weight": { |
| "residual": 0.020484907552599907, |
| "expected_noise": 0.021061100192360516 |
| }, |
| "model.language_model.layers.17.mlp.gate_proj.weight": { |
| "residual": 0.015057966113090515, |
| "expected_noise": 0.015311737544968045 |
| }, |
| "model.language_model.layers.17.mlp.up_proj.weight": { |
| "residual": 0.019757209345698357, |
| "expected_noise": 0.020332197892177016 |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023069006856530905, |
| "expected_noise": 0.020767078356491036 |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024264133535325527, |
| "expected_noise": 0.012101762511486223 |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.021955963224172592, |
| "expected_noise": 0.022796811695274054 |
| }, |
| "model.language_model.layers.18.linear_attn.in_proj_z.weight": { |
| "residual": 0.015344214625656605, |
| "expected_noise": 0.016003749876650368 |
| }, |
| "model.language_model.layers.18.linear_attn.out_proj.weight": { |
| "residual": 0.024049099534749985, |
| "expected_noise": 0.025440337660024636 |
| }, |
| "model.language_model.layers.18.mlp.down_proj.weight": { |
| "residual": 0.025966305285692215, |
| "expected_noise": 0.026902588588961825 |
| }, |
| "model.language_model.layers.18.mlp.gate_proj.weight": { |
| "residual": 0.014704960398375988, |
| "expected_noise": 0.014951469221847565 |
| }, |
| "model.language_model.layers.18.mlp.up_proj.weight": { |
| "residual": 0.019078686833381653, |
| "expected_noise": 0.019610966350702932 |
| }, |
| "model.language_model.layers.19.mlp.down_proj.weight": { |
| "residual": 0.02618139609694481, |
| "expected_noise": 0.027129116680386693 |
| }, |
| "model.language_model.layers.19.mlp.gate_proj.weight": { |
| "residual": 0.016629701480269432, |
| "expected_noise": 0.01700091681058303 |
| }, |
| "model.language_model.layers.19.mlp.up_proj.weight": { |
| "residual": 0.020547790452837944, |
| "expected_noise": 0.021167838390130086 |
| }, |
| "model.language_model.layers.19.self_attn.k_proj.weight": { |
| "residual": 0.018957659602165222, |
| "expected_noise": 0.02242538867050097 |
| }, |
| "model.language_model.layers.19.self_attn.o_proj.weight": { |
| "residual": 0.029659602791070938, |
| "expected_noise": 0.031475636937198924 |
| }, |
| "model.language_model.layers.19.self_attn.q_proj.weight": { |
| "residual": 0.022399073466658592, |
| "expected_noise": 0.023249055599899246 |
| }, |
| "model.language_model.layers.19.self_attn.v_proj.weight": { |
| "residual": 0.02949010767042637, |
| "expected_noise": 0.035086519017978796 |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023489007726311684, |
| "expected_noise": 0.026626236621778508 |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_b.weight": { |
| "residual": 0.002409006468951702, |
| "expected_noise": 0.0202388734368852 |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.021550627425312996, |
| "expected_noise": 0.022353139094194414 |
| }, |
| "model.language_model.layers.2.linear_attn.in_proj_z.weight": { |
| "residual": 0.01903187856078148, |
| "expected_noise": 0.020018051204809588 |
| }, |
| "model.language_model.layers.2.linear_attn.out_proj.weight": { |
| "residual": 0.027928804978728294, |
| "expected_noise": 0.029617494434947936 |
| }, |
| "model.language_model.layers.2.mlp.down_proj.weight": { |
| "residual": 0.027699677273631096, |
| "expected_noise": 0.028720948424611037 |
| }, |
| "model.language_model.layers.2.mlp.gate_proj.weight": { |
| "residual": 0.020510390400886536, |
| "expected_noise": 0.02112942816874963 |
| }, |
| "model.language_model.layers.2.mlp.up_proj.weight": { |
| "residual": 0.019041309133172035, |
| "expected_noise": 0.019572167074850943 |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_a.weight": { |
| "residual": 0.0024933740496635437, |
| "expected_noise": 0.012975840802365203 |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023669679649174213, |
| "expected_noise": 0.011099607234541002 |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.021638555452227592, |
| "expected_noise": 0.022461705170676662 |
| }, |
| "model.language_model.layers.20.linear_attn.in_proj_z.weight": { |
| "residual": 0.02150549367070198, |
| "expected_noise": 0.022700901010319685 |
| }, |
| "model.language_model.layers.20.linear_attn.out_proj.weight": { |
| "residual": 0.02467765100300312, |
| "expected_noise": 0.026113327798654145 |
| }, |
| "model.language_model.layers.20.mlp.down_proj.weight": { |
| "residual": 0.027928831055760384, |
| "expected_noise": 0.02897631971128551 |
| }, |
| "model.language_model.layers.20.mlp.gate_proj.weight": { |
| "residual": 0.017449302598834038, |
| "expected_noise": 0.01788317698008112 |
| }, |
| "model.language_model.layers.20.mlp.up_proj.weight": { |
| "residual": 0.020409924909472466, |
| "expected_noise": 0.02102553549186526 |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_a.weight": { |
| "residual": 0.002470360603183508, |
| "expected_noise": 0.019524069720159916 |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022954605519771576, |
| "expected_noise": 0.012622400592245734 |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.023453645408153534, |
| "expected_noise": 0.02439300840415004 |
| }, |
| "model.language_model.layers.21.linear_attn.in_proj_z.weight": { |
| "residual": 0.02118612267076969, |
| "expected_noise": 0.022347071262993744 |
| }, |
| "model.language_model.layers.21.linear_attn.out_proj.weight": { |
| "residual": 0.02356315217912197, |
| "expected_noise": 0.024910170483531145 |
| }, |
| "model.language_model.layers.21.mlp.down_proj.weight": { |
| "residual": 0.026752306148409843, |
| "expected_noise": 0.027737864078505387 |
| }, |
| "model.language_model.layers.21.mlp.gate_proj.weight": { |
| "residual": 0.017976487055420876, |
| "expected_noise": 0.018441225232188104 |
| }, |
| "model.language_model.layers.21.mlp.up_proj.weight": { |
| "residual": 0.018901705741882324, |
| "expected_noise": 0.019420092578360124 |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_a.weight": { |
| "residual": 0.0025535107124596834, |
| "expected_noise": 0.010929723419543826 |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_b.weight": { |
| "residual": 0.002692782785743475, |
| "expected_noise": 0.009283931392287146 |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.025397716090083122, |
| "expected_noise": 0.026447090624650983 |
| }, |
| "model.language_model.layers.22.linear_attn.in_proj_z.weight": { |
| "residual": 0.019652100279927254, |
| "expected_noise": 0.020685179268151496 |
| }, |
| "model.language_model.layers.22.linear_attn.out_proj.weight": { |
| "residual": 0.025643199682235718, |
| "expected_noise": 0.027152333785387176 |
| }, |
| "model.language_model.layers.22.mlp.down_proj.weight": { |
| "residual": 0.02758187986910343, |
| "expected_noise": 0.028616717792139913 |
| }, |
| "model.language_model.layers.22.mlp.gate_proj.weight": { |
| "residual": 0.02004006691277027, |
| "expected_noise": 0.02063230852142555 |
| }, |
| "model.language_model.layers.22.mlp.up_proj.weight": { |
| "residual": 0.01983746327459812, |
| "expected_noise": 0.02040704819363038 |
| }, |
| "model.language_model.layers.23.mlp.down_proj.weight": { |
| "residual": 0.030536841601133347, |
| "expected_noise": 0.031719459258292386 |
| }, |
| "model.language_model.layers.23.mlp.gate_proj.weight": { |
| "residual": 0.019045209512114525, |
| "expected_noise": 0.01958024951155318 |
| }, |
| "model.language_model.layers.23.mlp.up_proj.weight": { |
| "residual": 0.020957957953214645, |
| "expected_noise": 0.02159757829671833 |
| }, |
| "model.language_model.layers.23.self_attn.k_proj.weight": { |
| "residual": 0.019164014607667923, |
| "expected_noise": 0.02267273769656912 |
| }, |
| "model.language_model.layers.23.self_attn.o_proj.weight": { |
| "residual": 0.030518045648932457, |
| "expected_noise": 0.03241173259937376 |
| }, |
| "model.language_model.layers.23.self_attn.q_proj.weight": { |
| "residual": 0.013933026231825352, |
| "expected_noise": 0.014217769815189987 |
| }, |
| "model.language_model.layers.23.self_attn.v_proj.weight": { |
| "residual": 0.031782232224941254, |
| "expected_noise": 0.03780506386114819 |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_a.weight": { |
| "residual": 0.002303897403180599, |
| "expected_noise": 0.010547128379894506 |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022445039357990026, |
| "expected_noise": 0.006996909942560329 |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.024591149762272835, |
| "expected_noise": 0.025581205696665648 |
| }, |
| "model.language_model.layers.24.linear_attn.in_proj_z.weight": { |
| "residual": 0.019871653988957405, |
| "expected_noise": 0.02092720889536867 |
| }, |
| "model.language_model.layers.24.linear_attn.out_proj.weight": { |
| "residual": 0.023145200684666634, |
| "expected_noise": 0.024462332898404833 |
| }, |
| "model.language_model.layers.24.mlp.down_proj.weight": { |
| "residual": 0.03206007182598114, |
| "expected_noise": 0.03331711615406267 |
| }, |
| "model.language_model.layers.24.mlp.gate_proj.weight": { |
| "residual": 0.02031276747584343, |
| "expected_noise": 0.020929220497756914 |
| }, |
| "model.language_model.layers.24.mlp.up_proj.weight": { |
| "residual": 0.021343106403946877, |
| "expected_noise": 0.02200506382394341 |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023217187263071537, |
| "expected_noise": 0.015800398957043205 |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_b.weight": { |
| "residual": 0.00246169650927186, |
| "expected_noise": 0.013765278839865194 |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.025525745004415512, |
| "expected_noise": 0.02658210555406862 |
| }, |
| "model.language_model.layers.25.linear_attn.in_proj_z.weight": { |
| "residual": 0.02152973599731922, |
| "expected_noise": 0.022716639282890222 |
| }, |
| "model.language_model.layers.25.linear_attn.out_proj.weight": { |
| "residual": 0.027954382821917534, |
| "expected_noise": 0.029650370333532475 |
| }, |
| "model.language_model.layers.25.mlp.down_proj.weight": { |
| "residual": 0.03202132135629654, |
| "expected_noise": 0.033254834253246546 |
| }, |
| "model.language_model.layers.25.mlp.gate_proj.weight": { |
| "residual": 0.020470205694437027, |
| "expected_noise": 0.021097645243626166 |
| }, |
| "model.language_model.layers.25.mlp.up_proj.weight": { |
| "residual": 0.022038161754608154, |
| "expected_noise": 0.022744271305950362 |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_a.weight": { |
| "residual": 0.002825444098562002, |
| "expected_noise": 0.008031204739806447 |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022949634585529566, |
| "expected_noise": 0.007878500930504712 |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02575485035777092, |
| "expected_noise": 0.026828299690573543 |
| }, |
| "model.language_model.layers.26.linear_attn.in_proj_z.weight": { |
| "residual": 0.023783523589372635, |
| "expected_noise": 0.025152920994592615 |
| }, |
| "model.language_model.layers.26.linear_attn.out_proj.weight": { |
| "residual": 0.026964670047163963, |
| "expected_noise": 0.028582123548855403 |
| }, |
| "model.language_model.layers.26.mlp.down_proj.weight": { |
| "residual": 0.03295342996716499, |
| "expected_noise": 0.0342726318443205 |
| }, |
| "model.language_model.layers.26.mlp.gate_proj.weight": { |
| "residual": 0.02044123038649559, |
| "expected_noise": 0.021060999355802534 |
| }, |
| "model.language_model.layers.26.mlp.up_proj.weight": { |
| "residual": 0.022744642570614815, |
| "expected_noise": 0.02349245406507075 |
| }, |
| "model.language_model.layers.27.mlp.down_proj.weight": { |
| "residual": 0.0344485305249691, |
| "expected_noise": 0.03582923459412779 |
| }, |
| "model.language_model.layers.27.mlp.gate_proj.weight": { |
| "residual": 0.018758602440357208, |
| "expected_noise": 0.01926190959600742 |
| }, |
| "model.language_model.layers.27.mlp.up_proj.weight": { |
| "residual": 0.02329920418560505, |
| "expected_noise": 0.024085370986923885 |
| }, |
| "model.language_model.layers.27.self_attn.k_proj.weight": { |
| "residual": 0.018045084550976753, |
| "expected_noise": 0.02126691787556834 |
| }, |
| "model.language_model.layers.27.self_attn.o_proj.weight": { |
| "residual": 0.03532508388161659, |
| "expected_noise": 0.037595862379339 |
| }, |
| "model.language_model.layers.27.self_attn.q_proj.weight": { |
| "residual": 0.015709176659584045, |
| "expected_noise": 0.016122381504089782 |
| }, |
| "model.language_model.layers.27.self_attn.v_proj.weight": { |
| "residual": 0.03466886654496193, |
| "expected_noise": 0.04140207874827194 |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_a.weight": { |
| "residual": 0.002348849084228277, |
| "expected_noise": 0.015358139811271324 |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023300985340029, |
| "expected_noise": 0.01367995027283582 |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.027214843779802322, |
| "expected_noise": 0.028379857958831457 |
| }, |
| "model.language_model.layers.28.linear_attn.in_proj_z.weight": { |
| "residual": 0.019403886049985886, |
| "expected_noise": 0.020435632384124453 |
| }, |
| "model.language_model.layers.28.linear_attn.out_proj.weight": { |
| "residual": 0.026383696123957634, |
| "expected_noise": 0.02796489144592283 |
| }, |
| "model.language_model.layers.28.mlp.down_proj.weight": { |
| "residual": 0.03658055514097214, |
| "expected_noise": 0.03808082606174265 |
| }, |
| "model.language_model.layers.28.mlp.gate_proj.weight": { |
| "residual": 0.016489757224917412, |
| "expected_noise": 0.016862140017584213 |
| }, |
| "model.language_model.layers.28.mlp.up_proj.weight": { |
| "residual": 0.023742876946926117, |
| "expected_noise": 0.024553002887141146 |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_a.weight": { |
| "residual": 0.0021813029889017344, |
| "expected_noise": 0.01565512435130201 |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023673211690038443, |
| "expected_noise": 0.010617335914792449 |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02614693157374859, |
| "expected_noise": 0.027248967231877936 |
| }, |
| "model.language_model.layers.29.linear_attn.in_proj_z.weight": { |
| "residual": 0.02003108710050583, |
| "expected_noise": 0.02111177354527185 |
| }, |
| "model.language_model.layers.29.linear_attn.out_proj.weight": { |
| "residual": 0.02658865973353386, |
| "expected_noise": 0.028176674432110613 |
| }, |
| "model.language_model.layers.29.mlp.down_proj.weight": { |
| "residual": 0.02587934583425522, |
| "expected_noise": 0.026819870571929874 |
| }, |
| "model.language_model.layers.29.mlp.gate_proj.weight": { |
| "residual": 0.013650776818394661, |
| "expected_noise": 0.013824864974584036 |
| }, |
| "model.language_model.layers.29.mlp.up_proj.weight": { |
| "residual": 0.022128840908408165, |
| "expected_noise": 0.022850968301848896 |
| }, |
| "model.language_model.layers.3.mlp.down_proj.weight": { |
| "residual": 0.028187841176986694, |
| "expected_noise": 0.029240089765245312 |
| }, |
| "model.language_model.layers.3.mlp.gate_proj.weight": { |
| "residual": 0.020289288833737373, |
| "expected_noise": 0.02089489145828081 |
| }, |
| "model.language_model.layers.3.mlp.up_proj.weight": { |
| "residual": 0.019640956073999405, |
| "expected_noise": 0.02020588605846979 |
| }, |
| "model.language_model.layers.3.self_attn.k_proj.weight": { |
| "residual": 0.01940963976085186, |
| "expected_noise": 0.022941559171976933 |
| }, |
| "model.language_model.layers.3.self_attn.o_proj.weight": { |
| "residual": 0.025687796995043755, |
| "expected_noise": 0.027173730731762442 |
| }, |
| "model.language_model.layers.3.self_attn.q_proj.weight": { |
| "residual": 0.022417863830924034, |
| "expected_noise": 0.023303002299077512 |
| }, |
| "model.language_model.layers.3.self_attn.v_proj.weight": { |
| "residual": 0.017197713255882263, |
| "expected_noise": 0.020167400281900973 |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_a.weight": { |
| "residual": 0.002241450361907482, |
| "expected_noise": 0.011391209967009638 |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023068992886692286, |
| "expected_noise": 0.008783608276481681 |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.0216318741440773, |
| "expected_noise": 0.022451729382819235 |
| }, |
| "model.language_model.layers.30.linear_attn.in_proj_z.weight": { |
| "residual": 0.012920912355184555, |
| "expected_noise": 0.013350320068984579 |
| }, |
| "model.language_model.layers.30.linear_attn.out_proj.weight": { |
| "residual": 0.0261981263756752, |
| "expected_noise": 0.027780795299363382 |
| }, |
| "model.language_model.layers.30.mlp.down_proj.weight": { |
| "residual": 0.035306379199028015, |
| "expected_noise": 0.03670980385246957 |
| }, |
| "model.language_model.layers.30.mlp.gate_proj.weight": { |
| "residual": 0.01381426677107811, |
| "expected_noise": 0.013973329038787657 |
| }, |
| "model.language_model.layers.30.mlp.up_proj.weight": { |
| "residual": 0.015957843512296677, |
| "expected_noise": 0.01629212455564099 |
| }, |
| "model.language_model.layers.31.mlp.down_proj.weight": { |
| "residual": 0.0314432792365551, |
| "expected_noise": 0.032693815738486134 |
| }, |
| "model.language_model.layers.31.mlp.gate_proj.weight": { |
| "residual": 0.014272114261984825, |
| "expected_noise": 0.01449856611540217 |
| }, |
| "model.language_model.layers.31.mlp.up_proj.weight": { |
| "residual": 0.016216235235333443, |
| "expected_noise": 0.016572111082815057 |
| }, |
| "model.language_model.layers.31.self_attn.k_proj.weight": { |
| "residual": 0.015904834493994713, |
| "expected_noise": 0.018712307386911115 |
| }, |
| "model.language_model.layers.31.self_attn.o_proj.weight": { |
| "residual": 0.03138287365436554, |
| "expected_noise": 0.03335563231295409 |
| }, |
| "model.language_model.layers.31.self_attn.q_proj.weight": { |
| "residual": 0.013330838643014431, |
| "expected_noise": 0.013544674131252113 |
| }, |
| "model.language_model.layers.31.self_attn.v_proj.weight": { |
| "residual": 0.03287991136312485, |
| "expected_noise": 0.03918147284981273 |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023478625807911158, |
| "expected_noise": 0.02878515134461269 |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_b.weight": { |
| "residual": 0.002393813105300069, |
| "expected_noise": 0.019816150661547727 |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.023013344034552574, |
| "expected_noise": 0.02391927873736891 |
| }, |
| "model.language_model.layers.4.linear_attn.in_proj_z.weight": { |
| "residual": 0.02145136520266533, |
| "expected_noise": 0.022639228824619096 |
| }, |
| "model.language_model.layers.4.linear_attn.out_proj.weight": { |
| "residual": 0.027325760573148727, |
| "expected_noise": 0.028980732539427866 |
| }, |
| "model.language_model.layers.4.mlp.down_proj.weight": { |
| "residual": 0.02778732031583786, |
| "expected_noise": 0.028818107514189787 |
| }, |
| "model.language_model.layers.4.mlp.gate_proj.weight": { |
| "residual": 0.018016966059803963, |
| "expected_noise": 0.018481542806085157 |
| }, |
| "model.language_model.layers.4.mlp.up_proj.weight": { |
| "residual": 0.018945541232824326, |
| "expected_noise": 0.019458796594818376 |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_a.weight": { |
| "residual": 0.0021058013662695885, |
| "expected_noise": 0.023018095504582402 |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023318168241530657, |
| "expected_noise": 0.017309643591161356 |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02408265881240368, |
| "expected_noise": 0.025042537497620334 |
| }, |
| "model.language_model.layers.5.linear_attn.in_proj_z.weight": { |
| "residual": 0.01704740710556507, |
| "expected_noise": 0.01786151193731126 |
| }, |
| "model.language_model.layers.5.linear_attn.out_proj.weight": { |
| "residual": 0.024458350613713264, |
| "expected_noise": 0.02587277431555434 |
| }, |
| "model.language_model.layers.5.mlp.down_proj.weight": { |
| "residual": 0.02870899625122547, |
| "expected_noise": 0.029791411570998357 |
| }, |
| "model.language_model.layers.5.mlp.gate_proj.weight": { |
| "residual": 0.018459320068359375, |
| "expected_noise": 0.018952096953563168 |
| }, |
| "model.language_model.layers.5.mlp.up_proj.weight": { |
| "residual": 0.018651533871889114, |
| "expected_noise": 0.019156737281933116 |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_a.weight": { |
| "residual": 0.0023927378933876753, |
| "expected_noise": 0.02141772305748971 |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_b.weight": { |
| "residual": 0.0023017646744847298, |
| "expected_noise": 0.011666575968328263 |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02328074350953102, |
| "expected_noise": 0.02420788327951371 |
| }, |
| "model.language_model.layers.6.linear_attn.in_proj_z.weight": { |
| "residual": 0.017234593629837036, |
| "expected_noise": 0.018057453318839862 |
| }, |
| "model.language_model.layers.6.linear_attn.out_proj.weight": { |
| "residual": 0.027068275958299637, |
| "expected_noise": 0.028699695009793177 |
| }, |
| "model.language_model.layers.6.mlp.down_proj.weight": { |
| "residual": 0.028992246836423874, |
| "expected_noise": 0.03009146254689989 |
| }, |
| "model.language_model.layers.6.mlp.gate_proj.weight": { |
| "residual": 0.01833302713930607, |
| "expected_noise": 0.01882045866644678 |
| }, |
| "model.language_model.layers.6.mlp.up_proj.weight": { |
| "residual": 0.018001088872551918, |
| "expected_noise": 0.01847074521274833 |
| }, |
| "model.language_model.layers.7.mlp.down_proj.weight": { |
| "residual": 0.029047755524516106, |
| "expected_noise": 0.030155591803015826 |
| }, |
| "model.language_model.layers.7.mlp.gate_proj.weight": { |
| "residual": 0.019425256177783012, |
| "expected_noise": 0.019983638937361325 |
| }, |
| "model.language_model.layers.7.mlp.up_proj.weight": { |
| "residual": 0.018888941034674644, |
| "expected_noise": 0.01940782345684189 |
| }, |
| "model.language_model.layers.7.self_attn.k_proj.weight": { |
| "residual": 0.019985804334282875, |
| "expected_noise": 0.02373164356389053 |
| }, |
| "model.language_model.layers.7.self_attn.o_proj.weight": { |
| "residual": 0.0270765982568264, |
| "expected_noise": 0.028696284238971737 |
| }, |
| "model.language_model.layers.7.self_attn.q_proj.weight": { |
| "residual": 0.023774880915880203, |
| "expected_noise": 0.024736128632024724 |
| }, |
| "model.language_model.layers.7.self_attn.v_proj.weight": { |
| "residual": 0.02200007252395153, |
| "expected_noise": 0.02606835606988352 |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_a.weight": { |
| "residual": 0.002277798717841506, |
| "expected_noise": 0.02148478677450791 |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_b.weight": { |
| "residual": 0.0024263428058475256, |
| "expected_noise": 0.017478866539275866 |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.02486736699938774, |
| "expected_noise": 0.02587849786539514 |
| }, |
| "model.language_model.layers.8.linear_attn.in_proj_z.weight": { |
| "residual": 0.02005617320537567, |
| "expected_noise": 0.02113616466358378 |
| }, |
| "model.language_model.layers.8.linear_attn.out_proj.weight": { |
| "residual": 0.027381625026464462, |
| "expected_noise": 0.029035083587959826 |
| }, |
| "model.language_model.layers.8.mlp.down_proj.weight": { |
| "residual": 0.030264897271990776, |
| "expected_noise": 0.03144029514604296 |
| }, |
| "model.language_model.layers.8.mlp.gate_proj.weight": { |
| "residual": 0.019282547757029533, |
| "expected_noise": 0.019827030589280693 |
| }, |
| "model.language_model.layers.8.mlp.up_proj.weight": { |
| "residual": 0.01914236508309841, |
| "expected_noise": 0.01965182779387576 |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_a.weight": { |
| "residual": 0.002434649271890521, |
| "expected_noise": 0.020238299888668227 |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_b.weight": { |
| "residual": 0.0022778594866394997, |
| "expected_noise": 0.015301793525634018 |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": { |
| "residual": 0.027759335935115814, |
| "expected_noise": 0.028950258606694673 |
| }, |
| "model.language_model.layers.9.linear_attn.in_proj_z.weight": { |
| "residual": 0.02077699825167656, |
| "expected_noise": 0.02190708560015722 |
| }, |
| "model.language_model.layers.9.linear_attn.out_proj.weight": { |
| "residual": 0.027015967294573784, |
| "expected_noise": 0.028630227501949167 |
| }, |
| "model.language_model.layers.9.mlp.down_proj.weight": { |
| "residual": 0.03035132959485054, |
| "expected_noise": 0.0315272906552377 |
| }, |
| "model.language_model.layers.9.mlp.gate_proj.weight": { |
| "residual": 0.019494464620947838, |
| "expected_noise": 0.020056106672311842 |
| }, |
| "model.language_model.layers.9.mlp.up_proj.weight": { |
| "residual": 0.0195941012352705, |
| "expected_noise": 0.02015934420455228 |
| } |
| }, |
| "modules_to_save": [], |
| "full_deltas": [] |
| } |
