Blame for clef-flash/extraction.json

{
 "rank": 256,
 "max_residual": 0.05,
 "residuals": {
  "model.language_model.layers.0.linear_attn.in_proj_a.weight": {
   "residual": 0.002481166971847415,
   "expected_noise": 0.02831156518707557
  },
  "model.language_model.layers.0.linear_attn.in_proj_b.weight": {
   "residual": 0.0023871413432061672,
   "expected_noise": 0.008349564530849186
  },
  "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": {
   "residual": 0.028010139241814613,
   "expected_noise": 0.029223767926274215
  },
  "model.language_model.layers.0.linear_attn.in_proj_z.weight": {
   "residual": 0.028452618047595024,
   "expected_noise": 0.03020173782515911
  },
  "model.language_model.layers.0.linear_attn.out_proj.weight": {
   "residual": 0.020905354991555214,
   "expected_noise": 0.02208137960709428
  },
  "model.language_model.layers.0.mlp.down_proj.weight": {
   "residual": 0.028096821159124374,
   "expected_noise": 0.02914386182328274
  },
  "model.language_model.layers.0.mlp.gate_proj.weight": {
   "residual": 0.019913610070943832,
   "expected_noise": 0.020496658367335392
  },
  "model.language_model.layers.0.mlp.up_proj.weight": {
   "residual": 0.018833406269550323,
   "expected_noise": 0.019342525338178627
  },
  "model.language_model.layers.1.linear_attn.in_proj_a.weight": {
   "residual": 0.0021766992285847664,
   "expected_noise": 0.02457133336293675
  },
  "model.language_model.layers.1.linear_attn.in_proj_b.weight": {
   "residual": 0.0023660471197217703,
   "expected_noise": 0.025834611971101946
  },
  "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": {
   "residual": 0.020123932510614395,
   "expected_noise": 0.020844547236918608
  },
  "model.language_model.layers.1.linear_attn.in_proj_z.weight": {
   "residual": 0.017835108563303947,
   "expected_noise": 0.018704511759167593
  },
  "model.language_model.layers.1.linear_attn.out_proj.weight": {
   "residual": 0.027951691299676895,
   "expected_noise": 0.029649845699456806
  },
  "model.language_model.layers.1.mlp.down_proj.weight": {
   "residual": 0.028831562027335167,
   "expected_noise": 0.029920130335982904
  },
  "model.language_model.layers.1.mlp.gate_proj.weight": {
   "residual": 0.020388875156641006,
   "expected_noise": 0.02099758040634284
  },
  "model.language_model.layers.1.mlp.up_proj.weight": {
   "residual": 0.019373726099729538,
   "expected_noise": 0.019923920455313913
  },
  "model.language_model.layers.10.linear_attn.in_proj_a.weight": {
   "residual": 0.0020529604516923428,
   "expected_noise": 0.020798647550566877
  },
  "model.language_model.layers.10.linear_attn.in_proj_b.weight": {
   "residual": 0.002389292698353529,
   "expected_noise": 0.015135721668016834
  },
  "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": {
   "residual": 0.02783159725368023,
   "expected_noise": 0.029032253214033917
  },
  "model.language_model.layers.10.linear_attn.in_proj_z.weight": {
   "residual": 0.018996383994817734,
   "expected_noise": 0.01997516128233796
  },
  "model.language_model.layers.10.linear_attn.out_proj.weight": {
   "residual": 0.028022708371281624,
   "expected_noise": 0.029723322168313013
  },
  "model.language_model.layers.10.mlp.down_proj.weight": {
   "residual": 0.031969230622053146,
   "expected_noise": 0.03323809081162423
  },
  "model.language_model.layers.10.mlp.gate_proj.weight": {
   "residual": 0.019533751532435417,
   "expected_noise": 0.02009117624690135
  },
  "model.language_model.layers.10.mlp.up_proj.weight": {
   "residual": 0.020111756399273872,
   "expected_noise": 0.02071079010494138
  },
  "model.language_model.layers.11.mlp.down_proj.weight": {
   "residual": 0.031762782484292984,
   "expected_noise": 0.033010471921525224
  },
  "model.language_model.layers.11.mlp.gate_proj.weight": {
   "residual": 0.019560981541872025,
   "expected_noise": 0.020123060677130513
  },
  "model.language_model.layers.11.mlp.up_proj.weight": {
   "residual": 0.020869692787528038,
   "expected_noise": 0.02150906591768223
  },
  "model.language_model.layers.11.self_attn.k_proj.weight": {
   "residual": 0.022297900170087814,
   "expected_noise": 0.02651517617405365
  },
  "model.language_model.layers.11.self_attn.o_proj.weight": {
   "residual": 0.028615646064281464,
   "expected_noise": 0.030356000289846086
  },
  "model.language_model.layers.11.self_attn.q_proj.weight": {
   "residual": 0.025984665378928185,
   "expected_noise": 0.02706973672480417
  },
  "model.language_model.layers.11.self_attn.v_proj.weight": {
   "residual": 0.025407293811440468,
   "expected_noise": 0.03013468773536543
  },
  "model.language_model.layers.12.linear_attn.in_proj_a.weight": {
   "residual": 0.0024485806934535503,
   "expected_noise": 0.019420372968141388
  },
  "model.language_model.layers.12.linear_attn.in_proj_b.weight": {
   "residual": 0.002365445252507925,
   "expected_noise": 0.018309427496431052
  },
  "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": {
   "residual": 0.028319071978330612,
   "expected_noise": 0.029540904534968285
  },
  "model.language_model.layers.12.linear_attn.in_proj_z.weight": {
   "residual": 0.020267268642783165,
   "expected_noise": 0.02135990072261099
  },
  "model.language_model.layers.12.linear_attn.out_proj.weight": {
   "residual": 0.027437621727585793,
   "expected_noise": 0.02908597395512137
  },
  "model.language_model.layers.12.mlp.down_proj.weight": {
   "residual": 0.029705103486776352,
   "expected_noise": 0.030844441787333636
  },
  "model.language_model.layers.12.mlp.gate_proj.weight": {
   "residual": 0.0185822993516922,
   "expected_noise": 0.019082645358578235
  },
  "model.language_model.layers.12.mlp.up_proj.weight": {
   "residual": 0.02072763442993164,
   "expected_noise": 0.021360933630008464
  },
  "model.language_model.layers.13.linear_attn.in_proj_a.weight": {
   "residual": 0.002337323036044836,
   "expected_noise": 0.01972010034582101
  },
  "model.language_model.layers.13.linear_attn.in_proj_b.weight": {
   "residual": 0.002142813056707382,
   "expected_noise": 0.012867415629547428
  },
  "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": {
   "residual": 0.028196075931191444,
   "expected_noise": 0.02941212447972328
  },
  "model.language_model.layers.13.linear_attn.in_proj_z.weight": {
   "residual": 0.01853754185140133,
   "expected_noise": 0.019476448038852248
  },
  "model.language_model.layers.13.linear_attn.out_proj.weight": {
   "residual": 0.02737974189221859,
   "expected_noise": 0.02902977297271003
  },
  "model.language_model.layers.13.mlp.down_proj.weight": {
   "residual": 0.030016960576176643,
   "expected_noise": 0.03117935734213052
  },
  "model.language_model.layers.13.mlp.gate_proj.weight": {
   "residual": 0.018626024946570396,
   "expected_noise": 0.019128662090394866
  },
  "model.language_model.layers.13.mlp.up_proj.weight": {
   "residual": 0.02039594016969204,
   "expected_noise": 0.021011424876891414
  },
  "model.language_model.layers.14.linear_attn.in_proj_a.weight": {
   "residual": 0.0022698314860463142,
   "expected_noise": 0.01820066596925822
  },
  "model.language_model.layers.14.linear_attn.in_proj_b.weight": {
   "residual": 0.002186872297897935,
   "expected_noise": 0.011911765813455063
  },
  "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": {
   "residual": 0.027325650677084923,
   "expected_noise": 0.028500641387971117
  },
  "model.language_model.layers.14.linear_attn.in_proj_z.weight": {
   "residual": 0.017237689346075058,
   "expected_noise": 0.018069063693117022
  },
  "model.language_model.layers.14.linear_attn.out_proj.weight": {
   "residual": 0.027270525693893433,
   "expected_noise": 0.028916246566589286
  },
  "model.language_model.layers.14.mlp.down_proj.weight": {
   "residual": 0.031754836440086365,
   "expected_noise": 0.03300361054365933
  },
  "model.language_model.layers.14.mlp.gate_proj.weight": {
   "residual": 0.01796301081776619,
   "expected_noise": 0.018427800495340173
  },
  "model.language_model.layers.14.mlp.up_proj.weight": {
   "residual": 0.02063777483999729,
   "expected_noise": 0.02126145239538796
  },
  "model.language_model.layers.15.mlp.down_proj.weight": {
   "residual": 0.02903105691075325,
   "expected_noise": 0.030127538858782624
  },
  "model.language_model.layers.15.mlp.gate_proj.weight": {
   "residual": 0.01839308999478817,
   "expected_noise": 0.01887707116356956
  },
  "model.language_model.layers.15.mlp.up_proj.weight": {
   "residual": 0.020844554528594017,
   "expected_noise": 0.02148397871182622
  },
  "model.language_model.layers.15.self_attn.k_proj.weight": {
   "residual": 0.021484728902578354,
   "expected_noise": 0.02545731775315729
  },
  "model.language_model.layers.15.self_attn.o_proj.weight": {
   "residual": 0.02914666198194027,
   "expected_noise": 0.030932190911838176
  },
  "model.language_model.layers.15.self_attn.q_proj.weight": {
   "residual": 0.02431359328329563,
   "expected_noise": 0.025296783154912485
  },
  "model.language_model.layers.15.self_attn.v_proj.weight": {
   "residual": 0.029649870470166206,
   "expected_noise": 0.03523204815848145
  },
  "model.language_model.layers.16.linear_attn.in_proj_a.weight": {
   "residual": 0.002128089312463999,
   "expected_noise": 0.016310980441275585
  },
  "model.language_model.layers.16.linear_attn.in_proj_b.weight": {
   "residual": 0.0023869434371590614,
   "expected_noise": 0.010101458293557389
  },
  "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": {
   "residual": 0.028700554743409157,
   "expected_noise": 0.02994782085014652
  },
  "model.language_model.layers.16.linear_attn.in_proj_z.weight": {
   "residual": 0.019220247864723206,
   "expected_noise": 0.020221943151664038
  },
  "model.language_model.layers.16.linear_attn.out_proj.weight": {
   "residual": 0.02426838129758835,
   "expected_noise": 0.025670437003088648
  },
  "model.language_model.layers.16.mlp.down_proj.weight": {
   "residual": 0.022761620581150055,
   "expected_noise": 0.023513867209505185
  },
  "model.language_model.layers.16.mlp.gate_proj.weight": {
   "residual": 0.016623331233859062,
   "expected_noise": 0.01699752407774793
  },
  "model.language_model.layers.16.mlp.up_proj.weight": {
   "residual": 0.01988464966416359,
   "expected_noise": 0.020462601790522637
  },
  "model.language_model.layers.17.linear_attn.in_proj_a.weight": {
   "residual": 0.0024967638310045004,
   "expected_noise": 0.018321512866938
  },
  "model.language_model.layers.17.linear_attn.in_proj_b.weight": {
   "residual": 0.0022686421871185303,
   "expected_noise": 0.014988820124913314
  },
  "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": {
   "residual": 0.027975285425782204,
   "expected_noise": 0.029187326311521176
  },
  "model.language_model.layers.17.linear_attn.in_proj_z.weight": {
   "residual": 0.014796235598623753,
   "expected_noise": 0.015400105600467064
  },
  "model.language_model.layers.17.linear_attn.out_proj.weight": {
   "residual": 0.024706946685910225,
   "expected_noise": 0.026135422099318385
  },
  "model.language_model.layers.17.mlp.down_proj.weight": {
   "residual": 0.020484907552599907,
   "expected_noise": 0.021061100192360516
  },
  "model.language_model.layers.17.mlp.gate_proj.weight": {
   "residual": 0.015057966113090515,
   "expected_noise": 0.015311737544968045
  },
  "model.language_model.layers.17.mlp.up_proj.weight": {
   "residual": 0.019757209345698357,
   "expected_noise": 0.020332197892177016
  },
  "model.language_model.layers.18.linear_attn.in_proj_a.weight": {
   "residual": 0.0023069006856530905,
   "expected_noise": 0.020767078356491036
  },
  "model.language_model.layers.18.linear_attn.in_proj_b.weight": {
   "residual": 0.0024264133535325527,
   "expected_noise": 0.012101762511486223
  },
  "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": {
   "residual": 0.021955963224172592,
   "expected_noise": 0.022796811695274054
  },
  "model.language_model.layers.18.linear_attn.in_proj_z.weight": {
   "residual": 0.015344214625656605,
   "expected_noise": 0.016003749876650368
  },
  "model.language_model.layers.18.linear_attn.out_proj.weight": {
   "residual": 0.024049099534749985,
   "expected_noise": 0.025440337660024636
  },
  "model.language_model.layers.18.mlp.down_proj.weight": {
   "residual": 0.025966305285692215,
   "expected_noise": 0.026902588588961825
  },
  "model.language_model.layers.18.mlp.gate_proj.weight": {
   "residual": 0.014704960398375988,
   "expected_noise": 0.014951469221847565
  },
  "model.language_model.layers.18.mlp.up_proj.weight": {
   "residual": 0.019078686833381653,
   "expected_noise": 0.019610966350702932
  },
  "model.language_model.layers.19.mlp.down_proj.weight": {
   "residual": 0.02618139609694481,
   "expected_noise": 0.027129116680386693
  },
  "model.language_model.layers.19.mlp.gate_proj.weight": {
   "residual": 0.016629701480269432,
   "expected_noise": 0.01700091681058303
  },
  "model.language_model.layers.19.mlp.up_proj.weight": {
   "residual": 0.020547790452837944,
   "expected_noise": 0.021167838390130086
  },
  "model.language_model.layers.19.self_attn.k_proj.weight": {
   "residual": 0.018957659602165222,
   "expected_noise": 0.02242538867050097
  },
  "model.language_model.layers.19.self_attn.o_proj.weight": {
   "residual": 0.029659602791070938,
   "expected_noise": 0.031475636937198924
  },
  "model.language_model.layers.19.self_attn.q_proj.weight": {
   "residual": 0.022399073466658592,
   "expected_noise": 0.023249055599899246
  },
  "model.language_model.layers.19.self_attn.v_proj.weight": {
   "residual": 0.02949010767042637,
   "expected_noise": 0.035086519017978796
  },
  "model.language_model.layers.2.linear_attn.in_proj_a.weight": {
   "residual": 0.0023489007726311684,
   "expected_noise": 0.026626236621778508
  },
  "model.language_model.layers.2.linear_attn.in_proj_b.weight": {
   "residual": 0.002409006468951702,
   "expected_noise": 0.0202388734368852
  },
  "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": {
   "residual": 0.021550627425312996,
   "expected_noise": 0.022353139094194414
  },
  "model.language_model.layers.2.linear_attn.in_proj_z.weight": {
   "residual": 0.01903187856078148,
   "expected_noise": 0.020018051204809588
  },
  "model.language_model.layers.2.linear_attn.out_proj.weight": {
   "residual": 0.027928804978728294,
   "expected_noise": 0.029617494434947936
  },
  "model.language_model.layers.2.mlp.down_proj.weight": {
   "residual": 0.027699677273631096,
   "expected_noise": 0.028720948424611037
  },
  "model.language_model.layers.2.mlp.gate_proj.weight": {
   "residual": 0.020510390400886536,
   "expected_noise": 0.02112942816874963
  },
  "model.language_model.layers.2.mlp.up_proj.weight": {
   "residual": 0.019041309133172035,
   "expected_noise": 0.019572167074850943
  },
  "model.language_model.layers.20.linear_attn.in_proj_a.weight": {
   "residual": 0.0024933740496635437,
   "expected_noise": 0.012975840802365203
  },
  "model.language_model.layers.20.linear_attn.in_proj_b.weight": {
   "residual": 0.0023669679649174213,
   "expected_noise": 0.011099607234541002
  },
  "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": {
   "residual": 0.021638555452227592,
   "expected_noise": 0.022461705170676662
  },
  "model.language_model.layers.20.linear_attn.in_proj_z.weight": {
   "residual": 0.02150549367070198,
   "expected_noise": 0.022700901010319685
  },
  "model.language_model.layers.20.linear_attn.out_proj.weight": {
   "residual": 0.02467765100300312,
   "expected_noise": 0.026113327798654145
  },
  "model.language_model.layers.20.mlp.down_proj.weight": {
   "residual": 0.027928831055760384,
   "expected_noise": 0.02897631971128551
  },
  "model.language_model.layers.20.mlp.gate_proj.weight": {
   "residual": 0.017449302598834038,
   "expected_noise": 0.01788317698008112
  },
  "model.language_model.layers.20.mlp.up_proj.weight": {
   "residual": 0.020409924909472466,
   "expected_noise": 0.02102553549186526
  },
  "model.language_model.layers.21.linear_attn.in_proj_a.weight": {
   "residual": 0.002470360603183508,
   "expected_noise": 0.019524069720159916
  },
  "model.language_model.layers.21.linear_attn.in_proj_b.weight": {
   "residual": 0.0022954605519771576,
   "expected_noise": 0.012622400592245734
  },
  "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": {
   "residual": 0.023453645408153534,
   "expected_noise": 0.02439300840415004
  },
  "model.language_model.layers.21.linear_attn.in_proj_z.weight": {
   "residual": 0.02118612267076969,
   "expected_noise": 0.022347071262993744
  },
  "model.language_model.layers.21.linear_attn.out_proj.weight": {
   "residual": 0.02356315217912197,
   "expected_noise": 0.024910170483531145
  },
  "model.language_model.layers.21.mlp.down_proj.weight": {
   "residual": 0.026752306148409843,
   "expected_noise": 0.027737864078505387
  },
  "model.language_model.layers.21.mlp.gate_proj.weight": {
   "residual": 0.017976487055420876,
   "expected_noise": 0.018441225232188104
  },
  "model.language_model.layers.21.mlp.up_proj.weight": {
   "residual": 0.018901705741882324,
   "expected_noise": 0.019420092578360124
  },
  "model.language_model.layers.22.linear_attn.in_proj_a.weight": {
   "residual": 0.0025535107124596834,
   "expected_noise": 0.010929723419543826
  },
  "model.language_model.layers.22.linear_attn.in_proj_b.weight": {
   "residual": 0.002692782785743475,
   "expected_noise": 0.009283931392287146
  },
  "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": {
   "residual": 0.025397716090083122,
   "expected_noise": 0.026447090624650983
  },
  "model.language_model.layers.22.linear_attn.in_proj_z.weight": {
   "residual": 0.019652100279927254,
   "expected_noise": 0.020685179268151496
  },
  "model.language_model.layers.22.linear_attn.out_proj.weight": {
   "residual": 0.025643199682235718,
   "expected_noise": 0.027152333785387176
  },
  "model.language_model.layers.22.mlp.down_proj.weight": {
   "residual": 0.02758187986910343,
   "expected_noise": 0.028616717792139913
  },
  "model.language_model.layers.22.mlp.gate_proj.weight": {
   "residual": 0.02004006691277027,
   "expected_noise": 0.02063230852142555
  },
  "model.language_model.layers.22.mlp.up_proj.weight": {
   "residual": 0.01983746327459812,
   "expected_noise": 0.02040704819363038
  },
  "model.language_model.layers.23.mlp.down_proj.weight": {
   "residual": 0.030536841601133347,
   "expected_noise": 0.031719459258292386
  },
  "model.language_model.layers.23.mlp.gate_proj.weight": {
   "residual": 0.019045209512114525,
   "expected_noise": 0.01958024951155318
  },
  "model.language_model.layers.23.mlp.up_proj.weight": {
   "residual": 0.020957957953214645,
   "expected_noise": 0.02159757829671833
  },
  "model.language_model.layers.23.self_attn.k_proj.weight": {
   "residual": 0.019164014607667923,
   "expected_noise": 0.02267273769656912
  },
  "model.language_model.layers.23.self_attn.o_proj.weight": {
   "residual": 0.030518045648932457,
   "expected_noise": 0.03241173259937376
  },
  "model.language_model.layers.23.self_attn.q_proj.weight": {
   "residual": 0.013933026231825352,
   "expected_noise": 0.014217769815189987
  },
  "model.language_model.layers.23.self_attn.v_proj.weight": {
   "residual": 0.031782232224941254,
   "expected_noise": 0.03780506386114819
  },
  "model.language_model.layers.24.linear_attn.in_proj_a.weight": {
   "residual": 0.002303897403180599,
   "expected_noise": 0.010547128379894506
  },
  "model.language_model.layers.24.linear_attn.in_proj_b.weight": {
   "residual": 0.0022445039357990026,
   "expected_noise": 0.006996909942560329
  },
  "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": {
   "residual": 0.024591149762272835,
   "expected_noise": 0.025581205696665648
  },
  "model.language_model.layers.24.linear_attn.in_proj_z.weight": {
   "residual": 0.019871653988957405,
   "expected_noise": 0.02092720889536867
  },
  "model.language_model.layers.24.linear_attn.out_proj.weight": {
   "residual": 0.023145200684666634,
   "expected_noise": 0.024462332898404833
  },
  "model.language_model.layers.24.mlp.down_proj.weight": {
   "residual": 0.03206007182598114,
   "expected_noise": 0.03331711615406267
  },
  "model.language_model.layers.24.mlp.gate_proj.weight": {
   "residual": 0.02031276747584343,
   "expected_noise": 0.020929220497756914
  },
  "model.language_model.layers.24.mlp.up_proj.weight": {
   "residual": 0.021343106403946877,
   "expected_noise": 0.02200506382394341
  },
  "model.language_model.layers.25.linear_attn.in_proj_a.weight": {
   "residual": 0.0023217187263071537,
   "expected_noise": 0.015800398957043205
  },
  "model.language_model.layers.25.linear_attn.in_proj_b.weight": {
   "residual": 0.00246169650927186,
   "expected_noise": 0.013765278839865194
  },
  "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": {
   "residual": 0.025525745004415512,
   "expected_noise": 0.02658210555406862
  },
  "model.language_model.layers.25.linear_attn.in_proj_z.weight": {
   "residual": 0.02152973599731922,
   "expected_noise": 0.022716639282890222
  },
  "model.language_model.layers.25.linear_attn.out_proj.weight": {
   "residual": 0.027954382821917534,
   "expected_noise": 0.029650370333532475
  },
  "model.language_model.layers.25.mlp.down_proj.weight": {
   "residual": 0.03202132135629654,
   "expected_noise": 0.033254834253246546
  },
  "model.language_model.layers.25.mlp.gate_proj.weight": {
   "residual": 0.020470205694437027,
   "expected_noise": 0.021097645243626166
  },
  "model.language_model.layers.25.mlp.up_proj.weight": {
   "residual": 0.022038161754608154,
   "expected_noise": 0.022744271305950362
  },
  "model.language_model.layers.26.linear_attn.in_proj_a.weight": {
   "residual": 0.002825444098562002,
   "expected_noise": 0.008031204739806447
  },
  "model.language_model.layers.26.linear_attn.in_proj_b.weight": {
   "residual": 0.0022949634585529566,
   "expected_noise": 0.007878500930504712
  },
  "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": {
   "residual": 0.02575485035777092,
   "expected_noise": 0.026828299690573543
  },
  "model.language_model.layers.26.linear_attn.in_proj_z.weight": {
   "residual": 0.023783523589372635,
   "expected_noise": 0.025152920994592615
  },
  "model.language_model.layers.26.linear_attn.out_proj.weight": {
   "residual": 0.026964670047163963,
   "expected_noise": 0.028582123548855403
  },
  "model.language_model.layers.26.mlp.down_proj.weight": {
   "residual": 0.03295342996716499,
   "expected_noise": 0.0342726318443205
  },
  "model.language_model.layers.26.mlp.gate_proj.weight": {
   "residual": 0.02044123038649559,
   "expected_noise": 0.021060999355802534
  },
  "model.language_model.layers.26.mlp.up_proj.weight": {
   "residual": 0.022744642570614815,
   "expected_noise": 0.02349245406507075
  },
  "model.language_model.layers.27.mlp.down_proj.weight": {
   "residual": 0.0344485305249691,
   "expected_noise": 0.03582923459412779
  },
  "model.language_model.layers.27.mlp.gate_proj.weight": {
   "residual": 0.018758602440357208,
   "expected_noise": 0.01926190959600742
  },
  "model.language_model.layers.27.mlp.up_proj.weight": {
   "residual": 0.02329920418560505,
   "expected_noise": 0.024085370986923885
  },
  "model.language_model.layers.27.self_attn.k_proj.weight": {
   "residual": 0.018045084550976753,
   "expected_noise": 0.02126691787556834
  },
  "model.language_model.layers.27.self_attn.o_proj.weight": {
   "residual": 0.03532508388161659,
   "expected_noise": 0.037595862379339
  },
  "model.language_model.layers.27.self_attn.q_proj.weight": {
   "residual": 0.015709176659584045,
   "expected_noise": 0.016122381504089782
  },
  "model.language_model.layers.27.self_attn.v_proj.weight": {
   "residual": 0.03466886654496193,
   "expected_noise": 0.04140207874827194
  },
  "model.language_model.layers.28.linear_attn.in_proj_a.weight": {
   "residual": 0.002348849084228277,
   "expected_noise": 0.015358139811271324
  },
  "model.language_model.layers.28.linear_attn.in_proj_b.weight": {
   "residual": 0.0023300985340029,
   "expected_noise": 0.01367995027283582
  },
  "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": {
   "residual": 0.027214843779802322,
   "expected_noise": 0.028379857958831457
  },
  "model.language_model.layers.28.linear_attn.in_proj_z.weight": {
   "residual": 0.019403886049985886,
   "expected_noise": 0.020435632384124453
  },
  "model.language_model.layers.28.linear_attn.out_proj.weight": {
   "residual": 0.026383696123957634,
   "expected_noise": 0.02796489144592283
  },
  "model.language_model.layers.28.mlp.down_proj.weight": {
   "residual": 0.03658055514097214,
   "expected_noise": 0.03808082606174265
  },
  "model.language_model.layers.28.mlp.gate_proj.weight": {
   "residual": 0.016489757224917412,
   "expected_noise": 0.016862140017584213
  },
  "model.language_model.layers.28.mlp.up_proj.weight": {
   "residual": 0.023742876946926117,
   "expected_noise": 0.024553002887141146
  },
  "model.language_model.layers.29.linear_attn.in_proj_a.weight": {
   "residual": 0.0021813029889017344,
   "expected_noise": 0.01565512435130201
  },
  "model.language_model.layers.29.linear_attn.in_proj_b.weight": {
   "residual": 0.0023673211690038443,
   "expected_noise": 0.010617335914792449
  },
  "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": {
   "residual": 0.02614693157374859,
   "expected_noise": 0.027248967231877936
  },
  "model.language_model.layers.29.linear_attn.in_proj_z.weight": {
   "residual": 0.02003108710050583,
   "expected_noise": 0.02111177354527185
  },
  "model.language_model.layers.29.linear_attn.out_proj.weight": {
   "residual": 0.02658865973353386,
   "expected_noise": 0.028176674432110613
  },
  "model.language_model.layers.29.mlp.down_proj.weight": {
   "residual": 0.02587934583425522,
   "expected_noise": 0.026819870571929874
  },
  "model.language_model.layers.29.mlp.gate_proj.weight": {
   "residual": 0.013650776818394661,
   "expected_noise": 0.013824864974584036
  },
  "model.language_model.layers.29.mlp.up_proj.weight": {
   "residual": 0.022128840908408165,
   "expected_noise": 0.022850968301848896
  },
  "model.language_model.layers.3.mlp.down_proj.weight": {
   "residual": 0.028187841176986694,
   "expected_noise": 0.029240089765245312
  },
  "model.language_model.layers.3.mlp.gate_proj.weight": {
   "residual": 0.020289288833737373,
   "expected_noise": 0.02089489145828081
  },
  "model.language_model.layers.3.mlp.up_proj.weight": {
   "residual": 0.019640956073999405,
   "expected_noise": 0.02020588605846979
  },
  "model.language_model.layers.3.self_attn.k_proj.weight": {
   "residual": 0.01940963976085186,
   "expected_noise": 0.022941559171976933
  },
  "model.language_model.layers.3.self_attn.o_proj.weight": {
   "residual": 0.025687796995043755,
   "expected_noise": 0.027173730731762442
  },
  "model.language_model.layers.3.self_attn.q_proj.weight": {
   "residual": 0.022417863830924034,
   "expected_noise": 0.023303002299077512
  },
  "model.language_model.layers.3.self_attn.v_proj.weight": {
   "residual": 0.017197713255882263,
   "expected_noise": 0.020167400281900973
  },
  "model.language_model.layers.30.linear_attn.in_proj_a.weight": {
   "residual": 0.002241450361907482,
   "expected_noise": 0.011391209967009638
  },
  "model.language_model.layers.30.linear_attn.in_proj_b.weight": {
   "residual": 0.0023068992886692286,
   "expected_noise": 0.008783608276481681
  },
  "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": {
   "residual": 0.0216318741440773,
   "expected_noise": 0.022451729382819235
  },
  "model.language_model.layers.30.linear_attn.in_proj_z.weight": {
   "residual": 0.012920912355184555,
   "expected_noise": 0.013350320068984579
  },
  "model.language_model.layers.30.linear_attn.out_proj.weight": {
   "residual": 0.0261981263756752,
   "expected_noise": 0.027780795299363382
  },
  "model.language_model.layers.30.mlp.down_proj.weight": {
   "residual": 0.035306379199028015,
   "expected_noise": 0.03670980385246957
  },
  "model.language_model.layers.30.mlp.gate_proj.weight": {
   "residual": 0.01381426677107811,
   "expected_noise": 0.013973329038787657
  },
  "model.language_model.layers.30.mlp.up_proj.weight": {
   "residual": 0.015957843512296677,
   "expected_noise": 0.01629212455564099
  },
  "model.language_model.layers.31.mlp.down_proj.weight": {
   "residual": 0.0314432792365551,
   "expected_noise": 0.032693815738486134
  },
  "model.language_model.layers.31.mlp.gate_proj.weight": {
   "residual": 0.014272114261984825,
   "expected_noise": 0.01449856611540217
  },
  "model.language_model.layers.31.mlp.up_proj.weight": {
   "residual": 0.016216235235333443,
   "expected_noise": 0.016572111082815057
  },
  "model.language_model.layers.31.self_attn.k_proj.weight": {
   "residual": 0.015904834493994713,
   "expected_noise": 0.018712307386911115
  },
  "model.language_model.layers.31.self_attn.o_proj.weight": {
   "residual": 0.03138287365436554,
   "expected_noise": 0.03335563231295409
  },
  "model.language_model.layers.31.self_attn.q_proj.weight": {
   "residual": 0.013330838643014431,
   "expected_noise": 0.013544674131252113
  },
  "model.language_model.layers.31.self_attn.v_proj.weight": {
   "residual": 0.03287991136312485,
   "expected_noise": 0.03918147284981273
  },
  "model.language_model.layers.4.linear_attn.in_proj_a.weight": {
   "residual": 0.0023478625807911158,
   "expected_noise": 0.02878515134461269
  },
  "model.language_model.layers.4.linear_attn.in_proj_b.weight": {
   "residual": 0.002393813105300069,
   "expected_noise": 0.019816150661547727
  },
  "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": {
   "residual": 0.023013344034552574,
   "expected_noise": 0.02391927873736891
  },
  "model.language_model.layers.4.linear_attn.in_proj_z.weight": {
   "residual": 0.02145136520266533,
   "expected_noise": 0.022639228824619096
  },
  "model.language_model.layers.4.linear_attn.out_proj.weight": {
   "residual": 0.027325760573148727,
   "expected_noise": 0.028980732539427866
  },
  "model.language_model.layers.4.mlp.down_proj.weight": {
   "residual": 0.02778732031583786,
   "expected_noise": 0.028818107514189787
  },
  "model.language_model.layers.4.mlp.gate_proj.weight": {
   "residual": 0.018016966059803963,
   "expected_noise": 0.018481542806085157
  },
  "model.language_model.layers.4.mlp.up_proj.weight": {
   "residual": 0.018945541232824326,
   "expected_noise": 0.019458796594818376
  },
  "model.language_model.layers.5.linear_attn.in_proj_a.weight": {
   "residual": 0.0021058013662695885,
   "expected_noise": 0.023018095504582402
  },
  "model.language_model.layers.5.linear_attn.in_proj_b.weight": {
   "residual": 0.0023318168241530657,
   "expected_noise": 0.017309643591161356
  },
  "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": {
   "residual": 0.02408265881240368,
   "expected_noise": 0.025042537497620334
  },
  "model.language_model.layers.5.linear_attn.in_proj_z.weight": {
   "residual": 0.01704740710556507,
   "expected_noise": 0.01786151193731126
  },
  "model.language_model.layers.5.linear_attn.out_proj.weight": {
   "residual": 0.024458350613713264,
   "expected_noise": 0.02587277431555434
  },
  "model.language_model.layers.5.mlp.down_proj.weight": {
   "residual": 0.02870899625122547,
   "expected_noise": 0.029791411570998357
  },
  "model.language_model.layers.5.mlp.gate_proj.weight": {
   "residual": 0.018459320068359375,
   "expected_noise": 0.018952096953563168
  },
  "model.language_model.layers.5.mlp.up_proj.weight": {
   "residual": 0.018651533871889114,
   "expected_noise": 0.019156737281933116
  },
  "model.language_model.layers.6.linear_attn.in_proj_a.weight": {
   "residual": 0.0023927378933876753,
   "expected_noise": 0.02141772305748971
  },
  "model.language_model.layers.6.linear_attn.in_proj_b.weight": {
   "residual": 0.0023017646744847298,
   "expected_noise": 0.011666575968328263
  },
  "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": {
   "residual": 0.02328074350953102,
   "expected_noise": 0.02420788327951371
  },
  "model.language_model.layers.6.linear_attn.in_proj_z.weight": {
   "residual": 0.017234593629837036,
   "expected_noise": 0.018057453318839862
  },
  "model.language_model.layers.6.linear_attn.out_proj.weight": {
   "residual": 0.027068275958299637,
   "expected_noise": 0.028699695009793177
  },
  "model.language_model.layers.6.mlp.down_proj.weight": {
   "residual": 0.028992246836423874,
   "expected_noise": 0.03009146254689989
  },
  "model.language_model.layers.6.mlp.gate_proj.weight": {
   "residual": 0.01833302713930607,
   "expected_noise": 0.01882045866644678
  },
  "model.language_model.layers.6.mlp.up_proj.weight": {
   "residual": 0.018001088872551918,
   "expected_noise": 0.01847074521274833
  },
  "model.language_model.layers.7.mlp.down_proj.weight": {
   "residual": 0.029047755524516106,
   "expected_noise": 0.030155591803015826
  },
  "model.language_model.layers.7.mlp.gate_proj.weight": {
   "residual": 0.019425256177783012,
   "expected_noise": 0.019983638937361325
  },
  "model.language_model.layers.7.mlp.up_proj.weight": {
   "residual": 0.018888941034674644,
   "expected_noise": 0.01940782345684189
  },
  "model.language_model.layers.7.self_attn.k_proj.weight": {
   "residual": 0.019985804334282875,
   "expected_noise": 0.02373164356389053
  },
  "model.language_model.layers.7.self_attn.o_proj.weight": {
   "residual": 0.0270765982568264,
   "expected_noise": 0.028696284238971737
  },
  "model.language_model.layers.7.self_attn.q_proj.weight": {
   "residual": 0.023774880915880203,
   "expected_noise": 0.024736128632024724
  },
  "model.language_model.layers.7.self_attn.v_proj.weight": {
   "residual": 0.02200007252395153,
   "expected_noise": 0.02606835606988352
  },
  "model.language_model.layers.8.linear_attn.in_proj_a.weight": {
   "residual": 0.002277798717841506,
   "expected_noise": 0.02148478677450791
  },
  "model.language_model.layers.8.linear_attn.in_proj_b.weight": {
   "residual": 0.0024263428058475256,
   "expected_noise": 0.017478866539275866
  },
  "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": {
   "residual": 0.02486736699938774,
   "expected_noise": 0.02587849786539514
  },
  "model.language_model.layers.8.linear_attn.in_proj_z.weight": {
   "residual": 0.02005617320537567,
   "expected_noise": 0.02113616466358378
  },
  "model.language_model.layers.8.linear_attn.out_proj.weight": {
   "residual": 0.027381625026464462,
   "expected_noise": 0.029035083587959826
  },
  "model.language_model.layers.8.mlp.down_proj.weight": {
   "residual": 0.030264897271990776,
   "expected_noise": 0.03144029514604296
  },
  "model.language_model.layers.8.mlp.gate_proj.weight": {
   "residual": 0.019282547757029533,
   "expected_noise": 0.019827030589280693
  },
  "model.language_model.layers.8.mlp.up_proj.weight": {
   "residual": 0.01914236508309841,
   "expected_noise": 0.01965182779387576
  },
  "model.language_model.layers.9.linear_attn.in_proj_a.weight": {
   "residual": 0.002434649271890521,
   "expected_noise": 0.020238299888668227
  },
  "model.language_model.layers.9.linear_attn.in_proj_b.weight": {
   "residual": 0.0022778594866394997,
   "expected_noise": 0.015301793525634018
  },
  "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": {
   "residual": 0.027759335935115814,
   "expected_noise": 0.028950258606694673
  },
  "model.language_model.layers.9.linear_attn.in_proj_z.weight": {
   "residual": 0.02077699825167656,
   "expected_noise": 0.02190708560015722
  },
  "model.language_model.layers.9.linear_attn.out_proj.weight": {
   "residual": 0.027015967294573784,
   "expected_noise": 0.028630227501949167
  },
  "model.language_model.layers.9.mlp.down_proj.weight": {
   "residual": 0.03035132959485054,
   "expected_noise": 0.0315272906552377
  },
  "model.language_model.layers.9.mlp.gate_proj.weight": {
   "residual": 0.019494464620947838,
   "expected_noise": 0.020056106672311842
  },
  "model.language_model.layers.9.mlp.up_proj.weight": {
   "residual": 0.0195941012352705,
   "expected_noise": 0.02015934420455228
  }
 },
 "modules_to_save": [],
 "full_deltas": []
}