{ "schema_version": 1, "format": "comfy_scaled_fp8_merged_text_encoder", "scale_policy": "preserve_stock_per_tensor_scale", "base": "/workspace/Ideogram/checkpoints/text_encoders/qwen3vl_8b_fp8_scaled.safetensors", "base_sha256": "4ba424cf62e51392e4d1a39933e803706f4e823c1065f36aaf149c6453f66bcd", "adapter": "/workspace/Ideogram/artifacts/phase2_balanced_4000_8state_debanner_s020_v1_checkpoints/step_00001530/adapters.safetensors", "adapter_sha256": "eb53557a4d1476bf2b62ac6946476bc4fcbf8282034ab04df1e6a33feceaa96c", "adapter_global_step": "1530", "output": "/workspace/Ideogram/artifacts/huggingface_release/s020_v1/qwen3vl_8b_ideogram4_nl_s020_v1_step_00001530_fp8_scaled.safetensors", "output_bytes": 10588637800, "output_sha256": "d38a2669c13d2a602ed22be89b2f21bba5622cc6960b355ff0801c4d67489333", "output_tensors": 1254, "merged_projections": 252, "lora_rank": 64, "lora_alpha": 64.0, "elapsed_seconds": 62.515, "projections": [ { "index": 1, "total": 252, "projection": "layers.0.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4227, "parameters": 50331648, "reference_delta_l2": 3.1269727785306203, "retained_delta_l2": 2.847838946061438, "delta_retention_norm_ratio": 0.9107335265641971, "delta_cosine": 0.7260631937571582, "relative_weight_rmse": 0.012304272430821054 }, { "index": 2, "total": 252, "projection": "layers.0.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3802, "parameters": 50331648, "reference_delta_l2": 3.2955112311104933, "retained_delta_l2": 3.0060657864831555, "delta_retention_norm_ratio": 0.9121697896536062, "delta_cosine": 0.7246474927129162, "relative_weight_rmse": 0.013464760980937362 }, { "index": 3, "total": 252, "projection": "layers.0.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3827, "parameters": 50331648, "reference_delta_l2": 2.9185604335159008, "retained_delta_l2": 2.6044050786608244, "delta_retention_norm_ratio": 0.8923594826931086, "delta_cosine": 0.7051994882419824, "relative_weight_rmse": 0.012985052515934122 }, { "index": 4, "total": 252, "projection": "layers.0.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 0.6020524049592196, "retained_delta_l2": 0.4409459157004102, "delta_retention_norm_ratio": 0.7324045416449718, "delta_cosine": 0.5601915050421808, "relative_weight_rmse": 0.009005100813604077 }, { "index": 5, "total": 252, "projection": "layers.0.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.128, "parameters": 16777216, "reference_delta_l2": 1.973039220938326, "retained_delta_l2": 1.8611340842146211, "delta_retention_norm_ratio": 0.9432828625320049, "delta_cosine": 0.7627770951487235, "relative_weight_rmse": 0.013622675387227352 }, { "index": 6, "total": 252, "projection": "layers.0.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1267, "parameters": 16777216, "reference_delta_l2": 1.226169488957416, "retained_delta_l2": 0.9545936022842115, "delta_retention_norm_ratio": 0.7785168452494122, "delta_cosine": 0.5992496607567963, "relative_weight_rmse": 0.0098083098974002 }, { "index": 7, "total": 252, "projection": "layers.0.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.03, "parameters": 4194304, "reference_delta_l2": 0.7963862260294058, "retained_delta_l2": 0.693827574325755, "delta_retention_norm_ratio": 0.871219957915415, "delta_cosine": 0.6842739979149719, "relative_weight_rmse": 0.01199370525764303 }, { "index": 8, "total": 252, "projection": "layers.1.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4194, "parameters": 50331648, "reference_delta_l2": 2.0356480384900277, "retained_delta_l2": 1.9001821581406886, "delta_retention_norm_ratio": 0.9334531914221169, "delta_cosine": 0.7565319603551144, "relative_weight_rmse": 0.01197967566508188 }, { "index": 9, "total": 252, "projection": "layers.1.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3818, "parameters": 50331648, "reference_delta_l2": 3.8434880754811567, "retained_delta_l2": 3.810271658431912, "delta_retention_norm_ratio": 0.9913577416146696, "delta_cosine": 0.846772440107378, "relative_weight_rmse": 0.014363476876126656 }, { "index": 10, "total": 252, "projection": "layers.1.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3789, "parameters": 50331648, "reference_delta_l2": 4.22597100602001, "retained_delta_l2": 4.222919158295008, "delta_retention_norm_ratio": 0.9992778351482644, "delta_cosine": 0.9293241948032982, "relative_weight_rmse": 0.01576596702506838 }, { "index": 11, "total": 252, "projection": "layers.1.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 0.6774406044671824, "retained_delta_l2": 0.556723909165314, "delta_retention_norm_ratio": 0.8218047537956277, "delta_cosine": 0.6379560394473641, "relative_weight_rmse": 0.010828829915143327 }, { "index": 12, "total": 252, "projection": "layers.1.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1283, "parameters": 16777216, "reference_delta_l2": 1.7138713405131234, "retained_delta_l2": 1.5333545860507296, "delta_retention_norm_ratio": 0.8946731004858579, "delta_cosine": 0.7052913046292778, "relative_weight_rmse": 0.01266311458670879 }, { "index": 13, "total": 252, "projection": "layers.1.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1283, "parameters": 16777216, "reference_delta_l2": 2.0901680883206906, "retained_delta_l2": 2.0209289132881056, "delta_retention_norm_ratio": 0.966873872288322, "delta_cosine": 0.7924993655955894, "relative_weight_rmse": 0.01462637178641398 }, { "index": 14, "total": 252, "projection": "layers.1.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 0.9185446247120631, "retained_delta_l2": 0.8362166191359856, "delta_retention_norm_ratio": 0.910371251040868, "delta_cosine": 0.7229793123766166, "relative_weight_rmse": 0.012474231278458162 }, { "index": 15, "total": 252, "projection": "layers.10.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4217, "parameters": 50331648, "reference_delta_l2": 2.814992850391522, "retained_delta_l2": 2.4972698433859355, "delta_retention_norm_ratio": 0.8871318600466797, "delta_cosine": 0.7011173536136529, "relative_weight_rmse": 0.011668980634333866 }, { "index": 16, "total": 252, "projection": "layers.10.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3807, "parameters": 50331648, "reference_delta_l2": 2.3901047204142674, "retained_delta_l2": 1.8529896227829457, "delta_retention_norm_ratio": 0.7752754960718935, "delta_cosine": 0.5960595634147681, "relative_weight_rmse": 0.009651275667908231 }, { "index": 17, "total": 252, "projection": "layers.10.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3777, "parameters": 50331648, "reference_delta_l2": 2.858757490165862, "retained_delta_l2": 2.531018953398552, "delta_retention_norm_ratio": 0.8853562997579433, "delta_cosine": 0.6978358040396213, "relative_weight_rmse": 0.011913428049205213 }, { "index": 18, "total": 252, "projection": "layers.10.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0315, "parameters": 4194304, "reference_delta_l2": 0.8993867230288537, "retained_delta_l2": 0.8269512206391207, "delta_retention_norm_ratio": 0.9194612278177814, "delta_cosine": 0.7376595279985013, "relative_weight_rmse": 0.011978551081292697 }, { "index": 19, "total": 252, "projection": "layers.10.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1286, "parameters": 16777216, "reference_delta_l2": 1.7753697362828071, "retained_delta_l2": 1.5870460932219161, "delta_retention_norm_ratio": 0.8939242687243306, "delta_cosine": 0.7071171086611312, "relative_weight_rmse": 0.01217158891103583 }, { "index": 20, "total": 252, "projection": "layers.10.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1297, "parameters": 16777216, "reference_delta_l2": 2.1201857998503106, "retained_delta_l2": 2.0228005554894857, "delta_retention_norm_ratio": 0.9540675895632824, "delta_cosine": 0.7801449625303983, "relative_weight_rmse": 0.013454529664829615 }, { "index": 21, "total": 252, "projection": "layers.10.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 0.9083682746450251, "retained_delta_l2": 0.8106920352712357, "delta_retention_norm_ratio": 0.8924706618447681, "delta_cosine": 0.7108114019827055, "relative_weight_rmse": 0.011192079222170918 }, { "index": 22, "total": 252, "projection": "layers.11.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4228, "parameters": 50331648, "reference_delta_l2": 2.606977159285387, "retained_delta_l2": 2.2314210305488467, "delta_retention_norm_ratio": 0.8559419182485333, "delta_cosine": 0.6705462804874116, "relative_weight_rmse": 0.011040055210428465 }, { "index": 23, "total": 252, "projection": "layers.11.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3819, "parameters": 50331648, "reference_delta_l2": 2.7680849435848476, "retained_delta_l2": 2.3351837928526864, "delta_retention_norm_ratio": 0.8436098748575518, "delta_cosine": 0.6589891775775347, "relative_weight_rmse": 0.010796790721620953 }, { "index": 24, "total": 252, "projection": "layers.11.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3802, "parameters": 50331648, "reference_delta_l2": 2.6725192607048647, "retained_delta_l2": 2.270184682610497, "delta_retention_norm_ratio": 0.8494549378894826, "delta_cosine": 0.6631062521055323, "relative_weight_rmse": 0.01136743150604899 }, { "index": 25, "total": 252, "projection": "layers.11.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.1311720068959925, "retained_delta_l2": 1.0887236416233885, "delta_retention_norm_ratio": 0.9624739959848504, "delta_cosine": 0.7879357863041313, "relative_weight_rmse": 0.013664456675574239 }, { "index": 26, "total": 252, "projection": "layers.11.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1286, "parameters": 16777216, "reference_delta_l2": 1.6724912657557298, "retained_delta_l2": 1.4693522691368264, "delta_retention_norm_ratio": 0.8785410717663071, "delta_cosine": 0.693464586870248, "relative_weight_rmse": 0.011737459168859871 }, { "index": 27, "total": 252, "projection": "layers.11.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1291, "parameters": 16777216, "reference_delta_l2": 1.4032656533694052, "retained_delta_l2": 1.1662019028876682, "delta_retention_norm_ratio": 0.8310628141488968, "delta_cosine": 0.6459583221479873, "relative_weight_rmse": 0.010731881370672783 }, { "index": 28, "total": 252, "projection": "layers.11.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 0.9957992771310851, "retained_delta_l2": 0.9052954064265708, "delta_retention_norm_ratio": 0.9091143438412032, "delta_cosine": 0.7266247628209584, "relative_weight_rmse": 0.011870418153047967 }, { "index": 29, "total": 252, "projection": "layers.12.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4268, "parameters": 50331648, "reference_delta_l2": 2.5651561747652067, "retained_delta_l2": 2.1855692680213097, "delta_retention_norm_ratio": 0.8520219117736014, "delta_cosine": 0.6666850001099782, "relative_weight_rmse": 0.010824502316704015 }, { "index": 30, "total": 252, "projection": "layers.12.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3816, "parameters": 50331648, "reference_delta_l2": 3.204162211936139, "retained_delta_l2": 2.870834955746633, "delta_retention_norm_ratio": 0.895970542643629, "delta_cosine": 0.7120944191940738, "relative_weight_rmse": 0.011956951195376942 }, { "index": 31, "total": 252, "projection": "layers.12.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3783, "parameters": 50331648, "reference_delta_l2": 2.6816222918902204, "retained_delta_l2": 2.2846585181098966, "delta_retention_norm_ratio": 0.851968797029759, "delta_cosine": 0.665714734982328, "relative_weight_rmse": 0.011204452253251175 }, { "index": 32, "total": 252, "projection": "layers.12.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.9333935858291095, "retained_delta_l2": 0.8668283058369424, "delta_retention_norm_ratio": 0.92868466100178, "delta_cosine": 0.7480021997803925, "relative_weight_rmse": 0.012298593397018543 }, { "index": 33, "total": 252, "projection": "layers.12.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1283, "parameters": 16777216, "reference_delta_l2": 1.7763183189653498, "retained_delta_l2": 1.6100027453249657, "delta_retention_norm_ratio": 0.906370625205702, "delta_cosine": 0.722417460316628, "relative_weight_rmse": 0.012041667196134125 }, { "index": 34, "total": 252, "projection": "layers.12.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1291, "parameters": 16777216, "reference_delta_l2": 1.8083228859292346, "retained_delta_l2": 1.6539196082767817, "delta_retention_norm_ratio": 0.9146152056948001, "delta_cosine": 0.7315547943270726, "relative_weight_rmse": 0.012441787557432443 }, { "index": 35, "total": 252, "projection": "layers.12.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0301, "parameters": 4194304, "reference_delta_l2": 1.087080932464243, "retained_delta_l2": 1.0241777772412444, "delta_retention_norm_ratio": 0.9421357202168866, "delta_cosine": 0.7660598065202867, "relative_weight_rmse": 0.012191010035375172 }, { "index": 36, "total": 252, "projection": "layers.13.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.422, "parameters": 50331648, "reference_delta_l2": 2.3448327537056235, "retained_delta_l2": 1.9207321210034751, "delta_retention_norm_ratio": 0.8191339522906584, "delta_cosine": 0.6358730373758618, "relative_weight_rmse": 0.010190574670263465 }, { "index": 37, "total": 252, "projection": "layers.13.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3786, "parameters": 50331648, "reference_delta_l2": 2.9834700673737085, "retained_delta_l2": 2.6083816703705023, "delta_retention_norm_ratio": 0.8742778078771243, "delta_cosine": 0.6871331378438766, "relative_weight_rmse": 0.011869124678420157 }, { "index": 38, "total": 252, "projection": "layers.13.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3804, "parameters": 50331648, "reference_delta_l2": 2.401616101062306, "retained_delta_l2": 1.9418135391936848, "delta_retention_norm_ratio": 0.8085445206395657, "delta_cosine": 0.625121565212576, "relative_weight_rmse": 0.01034900613300074 }, { "index": 39, "total": 252, "projection": "layers.13.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.3429926619871464, "retained_delta_l2": 1.3290010352693131, "delta_retention_norm_ratio": 0.9895817549016754, "delta_cosine": 0.8348059277858658, "relative_weight_rmse": 0.014782896559259611 }, { "index": 40, "total": 252, "projection": "layers.13.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1291, "parameters": 16777216, "reference_delta_l2": 1.7282491063055783, "retained_delta_l2": 1.548538547686199, "delta_retention_norm_ratio": 0.8960158243602159, "delta_cosine": 0.711704845248182, "relative_weight_rmse": 0.012253301785092546 }, { "index": 41, "total": 252, "projection": "layers.13.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1283, "parameters": 16777216, "reference_delta_l2": 1.4115433572363378, "retained_delta_l2": 1.175948170162018, "delta_retention_norm_ratio": 0.8330939068456303, "delta_cosine": 0.6475618078393119, "relative_weight_rmse": 0.0107592151687993 }, { "index": 42, "total": 252, "projection": "layers.13.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 1.0309214075337587, "retained_delta_l2": 0.9500876562599483, "delta_retention_norm_ratio": 0.9215907724069999, "delta_cosine": 0.7391921596622596, "relative_weight_rmse": 0.012333454252584489 }, { "index": 43, "total": 252, "projection": "layers.14.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4185, "parameters": 50331648, "reference_delta_l2": 2.4700766817524675, "retained_delta_l2": 2.0896958078012076, "delta_retention_norm_ratio": 0.8460044270037043, "delta_cosine": 0.6616889399153113, "relative_weight_rmse": 0.010524003264311962 }, { "index": 44, "total": 252, "projection": "layers.14.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3786, "parameters": 50331648, "reference_delta_l2": 2.7277497199379104, "retained_delta_l2": 2.3124647233360016, "delta_retention_norm_ratio": 0.8477554617395904, "delta_cosine": 0.6610309572311588, "relative_weight_rmse": 0.011343492494406103 }, { "index": 45, "total": 252, "projection": "layers.14.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3784, "parameters": 50331648, "reference_delta_l2": 2.942592591945144, "retained_delta_l2": 2.6081569531417084, "delta_retention_norm_ratio": 0.8863466047869157, "delta_cosine": 0.7007091347828908, "relative_weight_rmse": 0.0117781189591218 }, { "index": 46, "total": 252, "projection": "layers.14.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 0.730438284908461, "retained_delta_l2": 0.6207823208583618, "delta_retention_norm_ratio": 0.8498764833173533, "delta_cosine": 0.6669530936392939, "relative_weight_rmse": 0.010731546950400846 }, { "index": 47, "total": 252, "projection": "layers.14.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1339, "parameters": 16777216, "reference_delta_l2": 1.5558795722316472, "retained_delta_l2": 1.3546514988750578, "delta_retention_norm_ratio": 0.8706660355030167, "delta_cosine": 0.6850243635581522, "relative_weight_rmse": 0.011440865436907418 }, { "index": 48, "total": 252, "projection": "layers.14.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.129, "parameters": 16777216, "reference_delta_l2": 2.328584458511773, "retained_delta_l2": 2.2710392840584004, "delta_retention_norm_ratio": 0.9752874866775711, "delta_cosine": 0.806257581943501, "relative_weight_rmse": 0.013951814180246073 }, { "index": 49, "total": 252, "projection": "layers.14.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 1.019341600551523, "retained_delta_l2": 0.9467884373671968, "delta_retention_norm_ratio": 0.9288235041667378, "delta_cosine": 0.7462463505114401, "relative_weight_rmse": 0.012397661612345819 }, { "index": 50, "total": 252, "projection": "layers.15.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4207, "parameters": 50331648, "reference_delta_l2": 2.3729892154991092, "retained_delta_l2": 1.9749539619015724, "delta_retention_norm_ratio": 0.8322641961464547, "delta_cosine": 0.6484044467838634, "relative_weight_rmse": 0.010304792086001663 }, { "index": 51, "total": 252, "projection": "layers.15.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3793, "parameters": 50331648, "reference_delta_l2": 2.860768519065232, "retained_delta_l2": 2.499002695722921, "delta_retention_norm_ratio": 0.8735424341636284, "delta_cosine": 0.6853882848453808, "relative_weight_rmse": 0.011983346093962125 }, { "index": 52, "total": 252, "projection": "layers.15.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3779, "parameters": 50331648, "reference_delta_l2": 2.784365351837079, "retained_delta_l2": 2.435335032959766, "delta_retention_norm_ratio": 0.8746463646923963, "delta_cosine": 0.6889917605823569, "relative_weight_rmse": 0.011411931564811607 }, { "index": 53, "total": 252, "projection": "layers.15.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.811292672406483, "retained_delta_l2": 0.7165957721306848, "delta_retention_norm_ratio": 0.883276524617306, "delta_cosine": 0.6981370952227917, "relative_weight_rmse": 0.011563410563958104 }, { "index": 54, "total": 252, "projection": "layers.15.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1281, "parameters": 16777216, "reference_delta_l2": 2.1524919824451088, "retained_delta_l2": 2.069290000539231, "delta_retention_norm_ratio": 0.9613462058932433, "delta_cosine": 0.7889175168869766, "relative_weight_rmse": 0.013845656978810577 }, { "index": 55, "total": 252, "projection": "layers.15.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1284, "parameters": 16777216, "reference_delta_l2": 2.237617525101801, "retained_delta_l2": 2.162694502621995, "delta_retention_norm_ratio": 0.9665166090096664, "delta_cosine": 0.8011403103935806, "relative_weight_rmse": 0.01345303681907185 }, { "index": 56, "total": 252, "projection": "layers.15.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 0.8555601598701333, "retained_delta_l2": 0.7601176861471938, "delta_retention_norm_ratio": 0.8884444622369666, "delta_cosine": 0.7049806055787962, "relative_weight_rmse": 0.011196324317514623 }, { "index": 57, "total": 252, "projection": "layers.16.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4219, "parameters": 50331648, "reference_delta_l2": 2.486670922049404, "retained_delta_l2": 2.123014037898322, "delta_retention_norm_ratio": 0.8537575354557281, "delta_cosine": 0.6684899230202213, "relative_weight_rmse": 0.010653798959174992 }, { "index": 58, "total": 252, "projection": "layers.16.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3825, "parameters": 50331648, "reference_delta_l2": 2.634456337737001, "retained_delta_l2": 2.250958051927088, "delta_retention_norm_ratio": 0.8544298190421565, "delta_cosine": 0.6679123511622501, "relative_weight_rmse": 0.011055786192365621 }, { "index": 59, "total": 252, "projection": "layers.16.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3819, "parameters": 50331648, "reference_delta_l2": 2.555779063907066, "retained_delta_l2": 2.154211406052585, "delta_retention_norm_ratio": 0.8428785713423143, "delta_cosine": 0.6569512870103185, "relative_weight_rmse": 0.0108856683839722 }, { "index": 60, "total": 252, "projection": "layers.16.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.7412285479580177, "retained_delta_l2": 0.6389193652031682, "delta_retention_norm_ratio": 0.8619734992173504, "delta_cosine": 0.6759999689935707, "relative_weight_rmse": 0.011222976218507576 }, { "index": 61, "total": 252, "projection": "layers.16.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1297, "parameters": 16777216, "reference_delta_l2": 1.6690605954864628, "retained_delta_l2": 1.493513315661502, "delta_retention_norm_ratio": 0.8948227042806698, "delta_cosine": 0.7098290927173299, "relative_weight_rmse": 0.011949817909595841 }, { "index": 62, "total": 252, "projection": "layers.16.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1301, "parameters": 16777216, "reference_delta_l2": 1.8097904473131718, "retained_delta_l2": 1.6776227535572412, "delta_retention_norm_ratio": 0.9269707197581092, "delta_cosine": 0.7479046228414, "relative_weight_rmse": 0.012462548760711567 }, { "index": 63, "total": 252, "projection": "layers.16.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0318, "parameters": 4194304, "reference_delta_l2": 0.9607306308156371, "retained_delta_l2": 0.8776427602291047, "delta_retention_norm_ratio": 0.913515955543134, "delta_cosine": 0.7303090433762676, "relative_weight_rmse": 0.01217207238997773 }, { "index": 64, "total": 252, "projection": "layers.17.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4235, "parameters": 50331648, "reference_delta_l2": 2.5743785829242163, "retained_delta_l2": 2.2126602193637552, "delta_retention_norm_ratio": 0.8594929409529237, "delta_cosine": 0.6740881397607131, "relative_weight_rmse": 0.010910443086156227 }, { "index": 65, "total": 252, "projection": "layers.17.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3837, "parameters": 50331648, "reference_delta_l2": 3.5206949864592643, "retained_delta_l2": 3.2974553312957817, "delta_retention_norm_ratio": 0.9365921626207123, "delta_cosine": 0.7552932233186679, "relative_weight_rmse": 0.013254577723690309 }, { "index": 66, "total": 252, "projection": "layers.17.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3789, "parameters": 50331648, "reference_delta_l2": 3.1737660104317835, "retained_delta_l2": 2.8884029937116837, "delta_retention_norm_ratio": 0.9100869390553222, "delta_cosine": 0.7246208492159082, "relative_weight_rmse": 0.012350866692247381 }, { "index": 67, "total": 252, "projection": "layers.17.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.7865736499704759, "retained_delta_l2": 0.7036271371727426, "delta_retention_norm_ratio": 0.8945470487082212, "delta_cosine": 0.7143540152366147, "relative_weight_rmse": 0.011553108870085779 }, { "index": 68, "total": 252, "projection": "layers.17.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1288, "parameters": 16777216, "reference_delta_l2": 1.7300110985450325, "retained_delta_l2": 1.5699751991256123, "delta_retention_norm_ratio": 0.907494293213487, "delta_cosine": 0.7246934984576693, "relative_weight_rmse": 0.012202281638474382 }, { "index": 69, "total": 252, "projection": "layers.17.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.131, "parameters": 16777216, "reference_delta_l2": 1.6625304841109203, "retained_delta_l2": 1.4882508413095632, "delta_retention_norm_ratio": 0.8951720618256468, "delta_cosine": 0.7081489443113639, "relative_weight_rmse": 0.011974792575036261 }, { "index": 70, "total": 252, "projection": "layers.17.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0307, "parameters": 4194304, "reference_delta_l2": 1.0397286647895199, "retained_delta_l2": 0.9760008642061044, "delta_retention_norm_ratio": 0.9387072774450089, "delta_cosine": 0.760020977587067, "relative_weight_rmse": 0.012815811763191241 }, { "index": 71, "total": 252, "projection": "layers.18.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4283, "parameters": 50331648, "reference_delta_l2": 2.907788266384756, "retained_delta_l2": 2.619001013143209, "delta_retention_norm_ratio": 0.9006849100465641, "delta_cosine": 0.7151994622396686, "relative_weight_rmse": 0.011707196901030538 }, { "index": 72, "total": 252, "projection": "layers.18.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3801, "parameters": 50331648, "reference_delta_l2": 4.695436688018443, "retained_delta_l2": 4.6538394778219105, "delta_retention_norm_ratio": 0.9911409283181948, "delta_cosine": 0.8260308652034198, "relative_weight_rmse": 0.015372444941754954 }, { "index": 73, "total": 252, "projection": "layers.18.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3802, "parameters": 50331648, "reference_delta_l2": 3.271060474066289, "retained_delta_l2": 3.010618542070301, "delta_retention_norm_ratio": 0.9203799703304689, "delta_cosine": 0.7369648534543821, "relative_weight_rmse": 0.01249584639898641 }, { "index": 74, "total": 252, "projection": "layers.18.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.1197355469645207, "retained_delta_l2": 1.0866039993052732, "delta_retention_norm_ratio": 0.9704112745647279, "delta_cosine": 0.7993672916496082, "relative_weight_rmse": 0.014013616857796325 }, { "index": 75, "total": 252, "projection": "layers.18.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1286, "parameters": 16777216, "reference_delta_l2": 1.630240547961089, "retained_delta_l2": 1.4574841013298292, "delta_retention_norm_ratio": 0.8940300884753952, "delta_cosine": 0.710921071578677, "relative_weight_rmse": 0.011706527987302644 }, { "index": 76, "total": 252, "projection": "layers.18.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1308, "parameters": 16777216, "reference_delta_l2": 2.1040520683613635, "retained_delta_l2": 2.01459798744349, "delta_retention_norm_ratio": 0.9574848539810423, "delta_cosine": 0.7843100071944246, "relative_weight_rmse": 0.013555611714405088 }, { "index": 77, "total": 252, "projection": "layers.18.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 0.9279404311746887, "retained_delta_l2": 0.8448875028231673, "delta_retention_norm_ratio": 0.9104975647559791, "delta_cosine": 0.7277625649326424, "relative_weight_rmse": 0.011786217011799544 }, { "index": 78, "total": 252, "projection": "layers.19.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4259, "parameters": 50331648, "reference_delta_l2": 2.6903627105142847, "retained_delta_l2": 2.353222050442629, "delta_retention_norm_ratio": 0.8746857965455489, "delta_cosine": 0.6889555065425643, "relative_weight_rmse": 0.01120078822653183 }, { "index": 79, "total": 252, "projection": "layers.19.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3795, "parameters": 50331648, "reference_delta_l2": 4.609492395721555, "retained_delta_l2": 4.542064055131354, "delta_retention_norm_ratio": 0.9853718512145097, "delta_cosine": 0.8182857698148852, "relative_weight_rmse": 0.015436989316013602 }, { "index": 80, "total": 252, "projection": "layers.19.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3798, "parameters": 50331648, "reference_delta_l2": 3.0876646446042115, "retained_delta_l2": 2.7671865882324376, "delta_retention_norm_ratio": 0.8962069741181836, "delta_cosine": 0.7104741013842515, "relative_weight_rmse": 0.012185078859367442 }, { "index": 81, "total": 252, "projection": "layers.19.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0322, "parameters": 4194304, "reference_delta_l2": 1.043740752861045, "retained_delta_l2": 1.0044144256684904, "delta_retention_norm_ratio": 0.9623217479199164, "delta_cosine": 0.7948396167245384, "relative_weight_rmse": 0.01359054135439678 }, { "index": 82, "total": 252, "projection": "layers.19.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1291, "parameters": 16777216, "reference_delta_l2": 1.6397315405573312, "retained_delta_l2": 1.4602018074618694, "delta_retention_norm_ratio": 0.8905127280564227, "delta_cosine": 0.7051211626469679, "relative_weight_rmse": 0.011795533568959942 }, { "index": 83, "total": 252, "projection": "layers.19.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.129, "parameters": 16777216, "reference_delta_l2": 1.939735345753661, "retained_delta_l2": 1.8307316810862433, "delta_retention_norm_ratio": 0.9438048778633429, "delta_cosine": 0.766157543535145, "relative_weight_rmse": 0.012834627407222321 }, { "index": 84, "total": 252, "projection": "layers.19.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 1.240107785319101, "retained_delta_l2": 1.2070027707450557, "delta_retention_norm_ratio": 0.9733047280519034, "delta_cosine": 0.8060440180768825, "relative_weight_rmse": 0.013912234594691633 }, { "index": 85, "total": 252, "projection": "layers.2.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4212, "parameters": 50331648, "reference_delta_l2": 1.5488637789895827, "retained_delta_l2": 1.333592944930879, "delta_retention_norm_ratio": 0.8610137076101438, "delta_cosine": 0.6978094443335896, "relative_weight_rmse": 0.010095503718382233 }, { "index": 86, "total": 252, "projection": "layers.2.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3797, "parameters": 50331648, "reference_delta_l2": 2.5067237100982185, "retained_delta_l2": 2.3633072007298566, "delta_retention_norm_ratio": 0.9427872689795788, "delta_cosine": 0.797306679939323, "relative_weight_rmse": 0.012650929505998188 }, { "index": 87, "total": 252, "projection": "layers.2.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3872, "parameters": 50331648, "reference_delta_l2": 3.388990795860513, "retained_delta_l2": 3.369597789494612, "delta_retention_norm_ratio": 0.9942776456077754, "delta_cosine": 0.8804779353007692, "relative_weight_rmse": 0.014975526320525897 }, { "index": 88, "total": 252, "projection": "layers.2.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0334, "parameters": 4194304, "reference_delta_l2": 0.5767551158284542, "retained_delta_l2": 0.4338007277480212, "delta_retention_norm_ratio": 0.7521402339447089, "delta_cosine": 0.5759507889214573, "relative_weight_rmse": 0.009284924835653236 }, { "index": 89, "total": 252, "projection": "layers.2.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1323, "parameters": 16777216, "reference_delta_l2": 2.0858322921045134, "retained_delta_l2": 1.9673083687243027, "delta_retention_norm_ratio": 0.9431766763661401, "delta_cosine": 0.760528198570256, "relative_weight_rmse": 0.013946873190318619 }, { "index": 90, "total": 252, "projection": "layers.2.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1292, "parameters": 16777216, "reference_delta_l2": 1.22645560287921, "retained_delta_l2": 0.9678507654384145, "delta_retention_norm_ratio": 0.7891445586503919, "delta_cosine": 0.6073761042200562, "relative_weight_rmse": 0.010166911918841 }, { "index": 91, "total": 252, "projection": "layers.2.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 1.160020130813952, "retained_delta_l2": 1.1187266586223268, "delta_retention_norm_ratio": 0.9644027969043514, "delta_cosine": 0.7930462437441694, "relative_weight_rmse": 0.013506082946072787 }, { "index": 92, "total": 252, "projection": "layers.20.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4185, "parameters": 50331648, "reference_delta_l2": 2.6417271050276083, "retained_delta_l2": 2.2537007027798666, "delta_retention_norm_ratio": 0.853116394381059, "delta_cosine": 0.6655062942081844, "relative_weight_rmse": 0.011234411585821199 }, { "index": 93, "total": 252, "projection": "layers.20.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3799, "parameters": 50331648, "reference_delta_l2": 4.493113904575293, "retained_delta_l2": 4.418058987551159, "delta_retention_norm_ratio": 0.9832955677024553, "delta_cosine": 0.8145858904153334, "relative_weight_rmse": 0.015326730744241561 }, { "index": 94, "total": 252, "projection": "layers.20.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3827, "parameters": 50331648, "reference_delta_l2": 3.1222963302444633, "retained_delta_l2": 2.8043110500175956, "delta_retention_norm_ratio": 0.8981565980311771, "delta_cosine": 0.7120258417330829, "relative_weight_rmse": 0.012265961759884383 }, { "index": 95, "total": 252, "projection": "layers.20.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 0.8398008136310411, "retained_delta_l2": 0.7713855239799262, "delta_retention_norm_ratio": 0.9185339088261797, "delta_cosine": 0.7387832950710411, "relative_weight_rmse": 0.01216287387936285 }, { "index": 96, "total": 252, "projection": "layers.20.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1305, "parameters": 16777216, "reference_delta_l2": 1.610349253641708, "retained_delta_l2": 1.4294397436336252, "delta_retention_norm_ratio": 0.8876582147636815, "delta_cosine": 0.7035696401886516, "relative_weight_rmse": 0.011828366034993853 }, { "index": 97, "total": 252, "projection": "layers.20.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1305, "parameters": 16777216, "reference_delta_l2": 1.8896686050855038, "retained_delta_l2": 1.7652411736320934, "delta_retention_norm_ratio": 0.9341538346361107, "delta_cosine": 0.7566278054862103, "relative_weight_rmse": 0.012326217639228183 }, { "index": 98, "total": 252, "projection": "layers.20.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0316, "parameters": 4194304, "reference_delta_l2": 0.943199112391037, "retained_delta_l2": 0.8653885918846724, "delta_retention_norm_ratio": 0.9175036113964181, "delta_cosine": 0.7368046799191541, "relative_weight_rmse": 0.012012097415746282 }, { "index": 99, "total": 252, "projection": "layers.21.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4227, "parameters": 50331648, "reference_delta_l2": 3.096352728706979, "retained_delta_l2": 2.7926921415794785, "delta_retention_norm_ratio": 0.9019295882177133, "delta_cosine": 0.7149317138374539, "relative_weight_rmse": 0.012284424447190342 }, { "index": 100, "total": 252, "projection": "layers.21.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3831, "parameters": 50331648, "reference_delta_l2": 4.601478745121303, "retained_delta_l2": 4.536254819602865, "delta_retention_norm_ratio": 0.9858254423998828, "delta_cosine": 0.8160791678831575, "relative_weight_rmse": 0.01586951141282905 }, { "index": 101, "total": 252, "projection": "layers.21.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3808, "parameters": 50331648, "reference_delta_l2": 3.3244217417021664, "retained_delta_l2": 3.048031819044838, "delta_retention_norm_ratio": 0.9168607522955822, "delta_cosine": 0.7318288858858512, "relative_weight_rmse": 0.0127188568650319 }, { "index": 102, "total": 252, "projection": "layers.21.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 0.8883500110394597, "retained_delta_l2": 0.821323965944344, "delta_retention_norm_ratio": 0.9245499586174504, "delta_cosine": 0.7418620870457034, "relative_weight_rmse": 0.012815217993680855 }, { "index": 103, "total": 252, "projection": "layers.21.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1284, "parameters": 16777216, "reference_delta_l2": 1.7363350952651777, "retained_delta_l2": 1.5879440262573505, "delta_retention_norm_ratio": 0.9145377701501999, "delta_cosine": 0.7322787696329526, "relative_weight_rmse": 0.01238042253526102 }, { "index": 104, "total": 252, "projection": "layers.21.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1298, "parameters": 16777216, "reference_delta_l2": 1.3431818757953342, "retained_delta_l2": 1.1096977110615218, "delta_retention_norm_ratio": 0.8261708492786503, "delta_cosine": 0.6430382021709762, "relative_weight_rmse": 0.010312170814682213 }, { "index": 105, "total": 252, "projection": "layers.21.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0302, "parameters": 4194304, "reference_delta_l2": 1.3401222621873143, "retained_delta_l2": 1.3232840437090565, "delta_retention_norm_ratio": 0.9874353117223984, "delta_cosine": 0.8233491223793983, "relative_weight_rmse": 0.0145521763231229 }, { "index": 106, "total": 252, "projection": "layers.22.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4182, "parameters": 50331648, "reference_delta_l2": 3.348253695373735, "retained_delta_l2": 3.1108121498557035, "delta_retention_norm_ratio": 0.9290849597669067, "delta_cosine": 0.7466446635867791, "relative_weight_rmse": 0.012576642698515664 }, { "index": 107, "total": 252, "projection": "layers.22.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3841, "parameters": 50331648, "reference_delta_l2": 4.191437704163066, "retained_delta_l2": 4.074424099330737, "delta_retention_norm_ratio": 0.9720827045297352, "delta_cosine": 0.8009936044647307, "relative_weight_rmse": 0.014613979432748203 }, { "index": 108, "total": 252, "projection": "layers.22.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3823, "parameters": 50331648, "reference_delta_l2": 3.4193685344857125, "retained_delta_l2": 3.1681149826609203, "delta_retention_norm_ratio": 0.9265204819864257, "delta_cosine": 0.7446025566787715, "relative_weight_rmse": 0.01260296600000183 }, { "index": 109, "total": 252, "projection": "layers.22.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 1.1233588468588183, "retained_delta_l2": 1.0978930305204808, "delta_retention_norm_ratio": 0.9773306486973901, "delta_cosine": 0.8174398193523686, "relative_weight_rmse": 0.014397857497982244 }, { "index": 110, "total": 252, "projection": "layers.22.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1292, "parameters": 16777216, "reference_delta_l2": 2.096172040574513, "retained_delta_l2": 1.9955246542014025, "delta_retention_norm_ratio": 0.951985149870845, "delta_cosine": 0.7726040268200726, "relative_weight_rmse": 0.01372359958810335 }, { "index": 111, "total": 252, "projection": "layers.22.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1295, "parameters": 16777216, "reference_delta_l2": 1.8735820710146767, "retained_delta_l2": 1.7449586608398842, "delta_retention_norm_ratio": 0.9313489319925368, "delta_cosine": 0.7552588343752609, "relative_weight_rmse": 0.012605390978871518 }, { "index": 112, "total": 252, "projection": "layers.22.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0298, "parameters": 4194304, "reference_delta_l2": 1.104789789987148, "retained_delta_l2": 1.0530537624706828, "delta_retention_norm_ratio": 0.9531711570967116, "delta_cosine": 0.7782066073504973, "relative_weight_rmse": 0.013219467528028158 }, { "index": 113, "total": 252, "projection": "layers.23.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4151, "parameters": 50331648, "reference_delta_l2": 3.1573986783340944, "retained_delta_l2": 2.8972070137585906, "delta_retention_norm_ratio": 0.9175930279692186, "delta_cosine": 0.7349246598047527, "relative_weight_rmse": 0.011965726104044899 }, { "index": 114, "total": 252, "projection": "layers.23.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3787, "parameters": 50331648, "reference_delta_l2": 5.5870157652651, "retained_delta_l2": 5.626138092141853, "delta_retention_norm_ratio": 1.007002365577699, "delta_cosine": 0.851880484960273, "relative_weight_rmse": 0.016744569548766984 }, { "index": 115, "total": 252, "projection": "layers.23.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.382, "parameters": 50331648, "reference_delta_l2": 4.955648869435568, "retained_delta_l2": 4.915349353509262, "delta_retention_norm_ratio": 0.9918679638150198, "delta_cosine": 0.8280099828088893, "relative_weight_rmse": 0.015368761690623919 }, { "index": 116, "total": 252, "projection": "layers.23.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 0.7949727976848021, "retained_delta_l2": 0.7075521488332062, "delta_retention_norm_ratio": 0.8900331569756965, "delta_cosine": 0.7056548515418034, "relative_weight_rmse": 0.011980711537248771 }, { "index": 117, "total": 252, "projection": "layers.23.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1286, "parameters": 16777216, "reference_delta_l2": 1.9611178356180183, "retained_delta_l2": 1.8341140758076107, "delta_retention_norm_ratio": 0.9352390980777633, "delta_cosine": 0.7547164028337787, "relative_weight_rmse": 0.012811001876643429 }, { "index": 118, "total": 252, "projection": "layers.23.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.129, "parameters": 16777216, "reference_delta_l2": 1.7866810382619926, "retained_delta_l2": 1.6194285043128502, "delta_retention_norm_ratio": 0.9063892601044008, "delta_cosine": 0.7240963533143272, "relative_weight_rmse": 0.012027247097238233 }, { "index": 119, "total": 252, "projection": "layers.23.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 0.9268902828092771, "retained_delta_l2": 0.8511800459627746, "delta_retention_norm_ratio": 0.9183180164355212, "delta_cosine": 0.7376135916253438, "relative_weight_rmse": 0.011724739325830702 }, { "index": 120, "total": 252, "projection": "layers.24.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4194, "parameters": 50331648, "reference_delta_l2": 3.248001361732338, "retained_delta_l2": 2.9759227463417086, "delta_retention_norm_ratio": 0.9162319885095382, "delta_cosine": 0.731467528974081, "relative_weight_rmse": 0.012265286006596474 }, { "index": 121, "total": 252, "projection": "layers.24.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3798, "parameters": 50331648, "reference_delta_l2": 5.079159790012967, "retained_delta_l2": 5.041899184536804, "delta_retention_norm_ratio": 0.9926640218034828, "delta_cosine": 0.8255122463800576, "relative_weight_rmse": 0.016177434591443086 }, { "index": 122, "total": 252, "projection": "layers.24.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3785, "parameters": 50331648, "reference_delta_l2": 4.002127200102585, "retained_delta_l2": 3.8233247575193747, "delta_retention_norm_ratio": 0.9553231485049684, "delta_cosine": 0.7753053899270538, "relative_weight_rmse": 0.013931788832164253 }, { "index": 123, "total": 252, "projection": "layers.24.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0311, "parameters": 4194304, "reference_delta_l2": 0.9638034365683473, "retained_delta_l2": 0.9140020321799122, "delta_retention_norm_ratio": 0.9483282560542068, "delta_cosine": 0.7718683986406603, "relative_weight_rmse": 0.013132119891595104 }, { "index": 124, "total": 252, "projection": "layers.24.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.128, "parameters": 16777216, "reference_delta_l2": 1.914972691650025, "retained_delta_l2": 1.7753847481146188, "delta_retention_norm_ratio": 0.9271070840100958, "delta_cosine": 0.7456926850988511, "relative_weight_rmse": 0.012568758413584789 }, { "index": 125, "total": 252, "projection": "layers.24.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1284, "parameters": 16777216, "reference_delta_l2": 3.0094068047413, "retained_delta_l2": 3.004943364043855, "delta_retention_norm_ratio": 0.9985168370422991, "delta_cosine": 0.8481736759550794, "relative_weight_rmse": 0.015521565684129443 }, { "index": 126, "total": 252, "projection": "layers.24.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0294, "parameters": 4194304, "reference_delta_l2": 1.0899244622607374, "retained_delta_l2": 1.039031573136485, "delta_retention_norm_ratio": 0.9533060401097068, "delta_cosine": 0.7795839023899411, "relative_weight_rmse": 0.013062099484726879 }, { "index": 127, "total": 252, "projection": "layers.25.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4275, "parameters": 50331648, "reference_delta_l2": 3.633931408786495, "retained_delta_l2": 3.400346578673652, "delta_retention_norm_ratio": 0.9357211780200205, "delta_cosine": 0.7512748559045608, "relative_weight_rmse": 0.013255124553773066 }, { "index": 128, "total": 252, "projection": "layers.25.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.382, "parameters": 50331648, "reference_delta_l2": 5.584503030567997, "retained_delta_l2": 5.596796432252929, "delta_retention_norm_ratio": 1.0022013421100573, "delta_cosine": 0.8400906306425101, "relative_weight_rmse": 0.016931961692208255 }, { "index": 129, "total": 252, "projection": "layers.25.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3794, "parameters": 50331648, "reference_delta_l2": 3.737413926592672, "retained_delta_l2": 3.4868664782660126, "delta_retention_norm_ratio": 0.9329623495690565, "delta_cosine": 0.7478532825383876, "relative_weight_rmse": 0.013602370535550844 }, { "index": 130, "total": 252, "projection": "layers.25.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 1.0272466552293884, "retained_delta_l2": 0.9806886758883855, "delta_retention_norm_ratio": 0.9546769229142866, "delta_cosine": 0.7797080882376601, "relative_weight_rmse": 0.013532519417488998 }, { "index": 131, "total": 252, "projection": "layers.25.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1281, "parameters": 16777216, "reference_delta_l2": 1.9527008465824647, "retained_delta_l2": 1.8050283587839864, "delta_retention_norm_ratio": 0.9243752630839852, "delta_cosine": 0.7397487044160916, "relative_weight_rmse": 0.01318621053304837 }, { "index": 132, "total": 252, "projection": "layers.25.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1289, "parameters": 16777216, "reference_delta_l2": 2.450536633916322, "retained_delta_l2": 2.4029435739095537, "delta_retention_norm_ratio": 0.9805785151921163, "delta_cosine": 0.8153367447804893, "relative_weight_rmse": 0.014591689306671735 }, { "index": 133, "total": 252, "projection": "layers.25.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0299, "parameters": 4194304, "reference_delta_l2": 1.185708349706415, "retained_delta_l2": 1.1420416359025602, "delta_retention_norm_ratio": 0.9631724666401593, "delta_cosine": 0.7868584096539307, "relative_weight_rmse": 0.01344112341912221 }, { "index": 134, "total": 252, "projection": "layers.26.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4184, "parameters": 50331648, "reference_delta_l2": 3.2898090617020626, "retained_delta_l2": 3.0098549097248215, "delta_retention_norm_ratio": 0.914902613882278, "delta_cosine": 0.7278429398237305, "relative_weight_rmse": 0.01232353824091038 }, { "index": 135, "total": 252, "projection": "layers.26.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3781, "parameters": 50331648, "reference_delta_l2": 5.26015590203999, "retained_delta_l2": 5.222621517173403, "delta_retention_norm_ratio": 0.992864396879942, "delta_cosine": 0.8222124191353445, "relative_weight_rmse": 0.016653677789210033 }, { "index": 136, "total": 252, "projection": "layers.26.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3846, "parameters": 50331648, "reference_delta_l2": 3.590069458954333, "retained_delta_l2": 3.316763413324665, "delta_retention_norm_ratio": 0.9238716551993196, "delta_cosine": 0.7375929700677254, "relative_weight_rmse": 0.01314661682087519 }, { "index": 137, "total": 252, "projection": "layers.26.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.032, "parameters": 4194304, "reference_delta_l2": 1.1020804593860358, "retained_delta_l2": 1.0637554874409914, "delta_retention_norm_ratio": 0.9652248875128454, "delta_cosine": 0.7913003888851425, "relative_weight_rmse": 0.014439016768790466 }, { "index": 138, "total": 252, "projection": "layers.26.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1279, "parameters": 16777216, "reference_delta_l2": 2.1431077000425294, "retained_delta_l2": 2.0182462766227127, "delta_retention_norm_ratio": 0.9417381481027113, "delta_cosine": 0.7600793909340787, "relative_weight_rmse": 0.013803256785379104 }, { "index": 139, "total": 252, "projection": "layers.26.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1277, "parameters": 16777216, "reference_delta_l2": 2.719952969878672, "retained_delta_l2": 2.696589776484013, "delta_retention_norm_ratio": 0.9914104421461003, "delta_cosine": 0.8326224887262295, "relative_weight_rmse": 0.015373628702985906 }, { "index": 140, "total": 252, "projection": "layers.26.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.03, "parameters": 4194304, "reference_delta_l2": 1.300620124236764, "retained_delta_l2": 1.2651561449286852, "delta_retention_norm_ratio": 0.972733022773356, "delta_cosine": 0.8066927122654023, "relative_weight_rmse": 0.01426096093375654 }, { "index": 141, "total": 252, "projection": "layers.27.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4175, "parameters": 50331648, "reference_delta_l2": 3.1127612376333307, "retained_delta_l2": 2.784529759790215, "delta_retention_norm_ratio": 0.8945529538614166, "delta_cosine": 0.706960318732661, "relative_weight_rmse": 0.011886623254813496 }, { "index": 142, "total": 252, "projection": "layers.27.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3814, "parameters": 50331648, "reference_delta_l2": 5.218309481668899, "retained_delta_l2": 5.184642749566007, "delta_retention_norm_ratio": 0.9935483450682336, "delta_cosine": 0.8246295446084377, "relative_weight_rmse": 0.016435645092332085 }, { "index": 143, "total": 252, "projection": "layers.27.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3802, "parameters": 50331648, "reference_delta_l2": 3.1371583611558593, "retained_delta_l2": 2.753390771113731, "delta_retention_norm_ratio": 0.8776703162983676, "delta_cosine": 0.6894176588930889, "relative_weight_rmse": 0.012165186836253738 }, { "index": 144, "total": 252, "projection": "layers.27.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0318, "parameters": 4194304, "reference_delta_l2": 0.9566006062182262, "retained_delta_l2": 0.9004468825838453, "delta_retention_norm_ratio": 0.9412986744213178, "delta_cosine": 0.7663775376414128, "relative_weight_rmse": 0.01348690912064083 }, { "index": 145, "total": 252, "projection": "layers.27.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.129, "parameters": 16777216, "reference_delta_l2": 1.8313920593686184, "retained_delta_l2": 1.6663060630742128, "delta_retention_norm_ratio": 0.9098576432883957, "delta_cosine": 0.723637594500945, "relative_weight_rmse": 0.01273018359232453 }, { "index": 146, "total": 252, "projection": "layers.27.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1278, "parameters": 16777216, "reference_delta_l2": 2.505769969980535, "retained_delta_l2": 2.4511339349255747, "delta_retention_norm_ratio": 0.9781959095569397, "delta_cosine": 0.8134119135818588, "relative_weight_rmse": 0.014897494871373889 }, { "index": 147, "total": 252, "projection": "layers.27.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0301, "parameters": 4194304, "reference_delta_l2": 1.3077336076957404, "retained_delta_l2": 1.2847687144409412, "delta_retention_norm_ratio": 0.9824391656529621, "delta_cosine": 0.8145542778588396, "relative_weight_rmse": 0.014327868666712023 }, { "index": 148, "total": 252, "projection": "layers.28.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4227, "parameters": 50331648, "reference_delta_l2": 3.753788447028095, "retained_delta_l2": 3.538348373928671, "delta_retention_norm_ratio": 0.9426072949662387, "delta_cosine": 0.7606475857732973, "relative_weight_rmse": 0.01308095261202953 }, { "index": 149, "total": 252, "projection": "layers.28.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3851, "parameters": 50331648, "reference_delta_l2": 6.384628728464536, "retained_delta_l2": 6.495170579397263, "delta_retention_norm_ratio": 1.0173137476952887, "delta_cosine": 0.8657629496281001, "relative_weight_rmse": 0.01790649059899249 }, { "index": 150, "total": 252, "projection": "layers.28.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3786, "parameters": 50331648, "reference_delta_l2": 3.236737353783521, "retained_delta_l2": 2.867916332999228, "delta_retention_norm_ratio": 0.8860516067659409, "delta_cosine": 0.6973045633320863, "relative_weight_rmse": 0.012307524760496754 }, { "index": 151, "total": 252, "projection": "layers.28.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 1.032288943294731, "retained_delta_l2": 0.9823748789222145, "delta_retention_norm_ratio": 0.9516471965560273, "delta_cosine": 0.7710451408030818, "relative_weight_rmse": 0.014138985070877371 }, { "index": 152, "total": 252, "projection": "layers.28.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1289, "parameters": 16777216, "reference_delta_l2": 1.8651147118672111, "retained_delta_l2": 1.7130419963641268, "delta_retention_norm_ratio": 0.918464685021523, "delta_cosine": 0.733442060333026, "relative_weight_rmse": 0.012824160612533261 }, { "index": 153, "total": 252, "projection": "layers.28.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1271, "parameters": 16777216, "reference_delta_l2": 2.105314988338305, "retained_delta_l2": 2.0052706387196078, "delta_retention_norm_ratio": 0.9524801038453345, "delta_cosine": 0.7768201327216553, "relative_weight_rmse": 0.013244203073642227 }, { "index": 154, "total": 252, "projection": "layers.28.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 1.057093412456726, "retained_delta_l2": 0.9882311455489668, "delta_retention_norm_ratio": 0.9348569709201757, "delta_cosine": 0.7521307203524212, "relative_weight_rmse": 0.012581262930721628 }, { "index": 155, "total": 252, "projection": "layers.29.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4232, "parameters": 50331648, "reference_delta_l2": 3.7318718982283765, "retained_delta_l2": 3.50050716333633, "delta_retention_norm_ratio": 0.93800303408006, "delta_cosine": 0.7553819010153049, "relative_weight_rmse": 0.012980868513796524 }, { "index": 156, "total": 252, "projection": "layers.29.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.382, "parameters": 50331648, "reference_delta_l2": 6.111839640398802, "retained_delta_l2": 6.194424768364596, "delta_retention_norm_ratio": 1.013512319174723, "delta_cosine": 0.8563557745027518, "relative_weight_rmse": 0.017696393880623638 }, { "index": 157, "total": 252, "projection": "layers.29.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3792, "parameters": 50331648, "reference_delta_l2": 3.574412188208577, "retained_delta_l2": 3.273411412573307, "delta_retention_norm_ratio": 0.9157901328145018, "delta_cosine": 0.7289595334981157, "relative_weight_rmse": 0.012884148207696703 }, { "index": 158, "total": 252, "projection": "layers.29.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 0.9613778159833816, "retained_delta_l2": 0.9108547548183007, "delta_retention_norm_ratio": 0.9474472363257088, "delta_cosine": 0.7814741530933297, "relative_weight_rmse": 0.013130078542218879 }, { "index": 159, "total": 252, "projection": "layers.29.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1272, "parameters": 16777216, "reference_delta_l2": 1.9429901539976733, "retained_delta_l2": 1.7908867120716394, "delta_retention_norm_ratio": 0.9217168231073722, "delta_cosine": 0.737270406241436, "relative_weight_rmse": 0.012918803561957477 }, { "index": 160, "total": 252, "projection": "layers.29.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1305, "parameters": 16777216, "reference_delta_l2": 2.940179549700112, "retained_delta_l2": 2.9265865240914684, "delta_retention_norm_ratio": 0.9953768042465193, "delta_cosine": 0.8475101752498819, "relative_weight_rmse": 0.015271158765365453 }, { "index": 161, "total": 252, "projection": "layers.29.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0298, "parameters": 4194304, "reference_delta_l2": 1.0698147715634443, "retained_delta_l2": 1.0104317699671235, "delta_retention_norm_ratio": 0.9444922586836808, "delta_cosine": 0.7676017515607246, "relative_weight_rmse": 0.012957966998483238 }, { "index": 162, "total": 252, "projection": "layers.3.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4196, "parameters": 50331648, "reference_delta_l2": 2.0469881021138128, "retained_delta_l2": 1.8067919142813011, "delta_retention_norm_ratio": 0.8826587279210494, "delta_cosine": 0.7129162638492356, "relative_weight_rmse": 0.011281224627702802 }, { "index": 163, "total": 252, "projection": "layers.3.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3811, "parameters": 50331648, "reference_delta_l2": 2.922833700925656, "retained_delta_l2": 2.701297923257113, "delta_retention_norm_ratio": 0.9242051377748987, "delta_cosine": 0.7456390004606592, "relative_weight_rmse": 0.01225232535469193 }, { "index": 164, "total": 252, "projection": "layers.3.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3803, "parameters": 50331648, "reference_delta_l2": 3.170575036365232, "retained_delta_l2": 3.1039285920713064, "delta_retention_norm_ratio": 0.9789796981526954, "delta_cosine": 0.8417859713352379, "relative_weight_rmse": 0.014662548426467735 }, { "index": 165, "total": 252, "projection": "layers.3.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 0.5928153003252811, "retained_delta_l2": 0.45925093149985696, "delta_retention_norm_ratio": 0.7746948016487823, "delta_cosine": 0.5960206976001503, "relative_weight_rmse": 0.009616935234874856 }, { "index": 166, "total": 252, "projection": "layers.3.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1286, "parameters": 16777216, "reference_delta_l2": 1.8492563116636087, "retained_delta_l2": 1.6954448559515825, "delta_retention_norm_ratio": 0.9168252368576988, "delta_cosine": 0.7302789316434527, "relative_weight_rmse": 0.012809784802966738 }, { "index": 167, "total": 252, "projection": "layers.3.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1293, "parameters": 16777216, "reference_delta_l2": 1.506985172201776, "retained_delta_l2": 1.3041041112765286, "delta_retention_norm_ratio": 0.8653728884214378, "delta_cosine": 0.6787827891036176, "relative_weight_rmse": 0.011717372717298042 }, { "index": 168, "total": 252, "projection": "layers.3.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0303, "parameters": 4194304, "reference_delta_l2": 0.9484510766660493, "retained_delta_l2": 0.8550643812206361, "delta_retention_norm_ratio": 0.9015376778592716, "delta_cosine": 0.7159253627656402, "relative_weight_rmse": 0.012402602443049677 }, { "index": 169, "total": 252, "projection": "layers.30.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4241, "parameters": 50331648, "reference_delta_l2": 3.881081942741242, "retained_delta_l2": 3.679213410037991, "delta_retention_norm_ratio": 0.9479865316730031, "delta_cosine": 0.7660717988774276, "relative_weight_rmse": 0.013128896619984549 }, { "index": 170, "total": 252, "projection": "layers.30.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3852, "parameters": 50331648, "reference_delta_l2": 5.385688107680878, "retained_delta_l2": 5.381089580955163, "delta_retention_norm_ratio": 0.9991461579961981, "delta_cosine": 0.8302506979760844, "relative_weight_rmse": 0.016908597417800836 }, { "index": 171, "total": 252, "projection": "layers.30.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3793, "parameters": 50331648, "reference_delta_l2": 3.7886798447697156, "retained_delta_l2": 3.5044195098186384, "delta_retention_norm_ratio": 0.924971138602936, "delta_cosine": 0.7381686108121072, "relative_weight_rmse": 0.01336324497183794 }, { "index": 172, "total": 252, "projection": "layers.30.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0321, "parameters": 4194304, "reference_delta_l2": 1.4967965365370788, "retained_delta_l2": 1.5084521827425899, "delta_retention_norm_ratio": 1.007787061180992, "delta_cosine": 0.8529388802167588, "relative_weight_rmse": 0.01661322374914887 }, { "index": 173, "total": 252, "projection": "layers.30.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1293, "parameters": 16777216, "reference_delta_l2": 1.8641895953134608, "retained_delta_l2": 1.687691231712539, "delta_retention_norm_ratio": 0.9053216668279688, "delta_cosine": 0.7170319774495255, "relative_weight_rmse": 0.01284219142237025 }, { "index": 174, "total": 252, "projection": "layers.30.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1265, "parameters": 16777216, "reference_delta_l2": 3.01325671683292, "retained_delta_l2": 3.0238985865702173, "delta_retention_norm_ratio": 1.003531683735358, "delta_cosine": 0.854792895207731, "relative_weight_rmse": 0.01627868799924678 }, { "index": 175, "total": 252, "projection": "layers.30.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 1.3092454472265989, "retained_delta_l2": 1.2711618368957232, "delta_retention_norm_ratio": 0.9709117870819799, "delta_cosine": 0.8013761680772169, "relative_weight_rmse": 0.013920487605929862 }, { "index": 176, "total": 252, "projection": "layers.31.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4169, "parameters": 50331648, "reference_delta_l2": 4.1084550327590295, "retained_delta_l2": 3.9426855677034225, "delta_retention_norm_ratio": 0.9596516296919807, "delta_cosine": 0.7804069254305991, "relative_weight_rmse": 0.013375184816653753 }, { "index": 177, "total": 252, "projection": "layers.31.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.384, "parameters": 50331648, "reference_delta_l2": 4.204636558660181, "retained_delta_l2": 4.054791818793385, "delta_retention_norm_ratio": 0.9643620232625897, "delta_cosine": 0.7852028074197616, "relative_weight_rmse": 0.014739784107740565 }, { "index": 178, "total": 252, "projection": "layers.31.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3806, "parameters": 50331648, "reference_delta_l2": 3.7410814813654816, "retained_delta_l2": 3.452811685816432, "delta_retention_norm_ratio": 0.922944796314933, "delta_cosine": 0.7364122265675088, "relative_weight_rmse": 0.01311554283213831 }, { "index": 179, "total": 252, "projection": "layers.31.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 1.1323445033739201, "retained_delta_l2": 1.1086924409108212, "delta_retention_norm_ratio": 0.9791123086722939, "delta_cosine": 0.8130617153573887, "relative_weight_rmse": 0.014444677927331962 }, { "index": 180, "total": 252, "projection": "layers.31.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1322, "parameters": 16777216, "reference_delta_l2": 1.9231706062594072, "retained_delta_l2": 1.7740289106067064, "delta_retention_norm_ratio": 0.9224500961239297, "delta_cosine": 0.7383528409683014, "relative_weight_rmse": 0.01252349737431234 }, { "index": 181, "total": 252, "projection": "layers.31.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1269, "parameters": 16777216, "reference_delta_l2": 3.0864807513581716, "retained_delta_l2": 3.1091758585622844, "delta_retention_norm_ratio": 1.007353069412186, "delta_cosine": 0.8588801820594556, "relative_weight_rmse": 0.01633773267765145 }, { "index": 182, "total": 252, "projection": "layers.31.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 0.911606139216967, "retained_delta_l2": 0.8037103391056234, "delta_retention_norm_ratio": 0.8816420870047872, "delta_cosine": 0.698835520254118, "relative_weight_rmse": 0.011235475360537445 }, { "index": 183, "total": 252, "projection": "layers.32.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4267, "parameters": 50331648, "reference_delta_l2": 3.0495582576332665, "retained_delta_l2": 2.6967207646394122, "delta_retention_norm_ratio": 0.8842988186532668, "delta_cosine": 0.6980275932606044, "relative_weight_rmse": 0.01127934914427212 }, { "index": 184, "total": 252, "projection": "layers.32.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.382, "parameters": 50331648, "reference_delta_l2": 5.692511022736389, "retained_delta_l2": 5.739681946755911, "delta_retention_norm_ratio": 1.0082864879542819, "delta_cosine": 0.8468962010552334, "relative_weight_rmse": 0.017428843321927585 }, { "index": 185, "total": 252, "projection": "layers.32.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3842, "parameters": 50331648, "reference_delta_l2": 4.1613233155376745, "retained_delta_l2": 3.9427416482145152, "delta_retention_norm_ratio": 0.9474730390433705, "delta_cosine": 0.7663476507769479, "relative_weight_rmse": 0.013800835891177412 }, { "index": 186, "total": 252, "projection": "layers.32.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0325, "parameters": 4194304, "reference_delta_l2": 1.0749308514731222, "retained_delta_l2": 1.0404449056170206, "delta_retention_norm_ratio": 0.9679179867162238, "delta_cosine": 0.7986748314670193, "relative_weight_rmse": 0.014078454901238931 }, { "index": 187, "total": 252, "projection": "layers.32.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1296, "parameters": 16777216, "reference_delta_l2": 1.9521065727725724, "retained_delta_l2": 1.794087986749361, "delta_retention_norm_ratio": 0.9190522750001411, "delta_cosine": 0.7326878381179381, "relative_weight_rmse": 0.012903346914620015 }, { "index": 188, "total": 252, "projection": "layers.32.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1304, "parameters": 16777216, "reference_delta_l2": 3.489609887071843, "retained_delta_l2": 3.54785274581623, "delta_retention_norm_ratio": 1.0166903638599154, "delta_cosine": 0.8830956089103636, "relative_weight_rmse": 0.016991135952278535 }, { "index": 189, "total": 252, "projection": "layers.32.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.1042808294327686, "retained_delta_l2": 1.034127379772928, "delta_retention_norm_ratio": 0.9364713687043936, "delta_cosine": 0.7578150313742703, "relative_weight_rmse": 0.012600668825086671 }, { "index": 190, "total": 252, "projection": "layers.33.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4231, "parameters": 50331648, "reference_delta_l2": 4.666259945277719, "retained_delta_l2": 4.578391719038913, "delta_retention_norm_ratio": 0.9811694532089389, "delta_cosine": 0.8093000623021638, "relative_weight_rmse": 0.014381902234994248 }, { "index": 191, "total": 252, "projection": "layers.33.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3827, "parameters": 50331648, "reference_delta_l2": 3.9832469507356367, "retained_delta_l2": 3.7956235365409667, "delta_retention_norm_ratio": 0.9528968661709465, "delta_cosine": 0.7697364410652019, "relative_weight_rmse": 0.014686245571898645 }, { "index": 192, "total": 252, "projection": "layers.33.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3804, "parameters": 50331648, "reference_delta_l2": 4.536695712695747, "retained_delta_l2": 4.397943606116111, "delta_retention_norm_ratio": 0.969415602154814, "delta_cosine": 0.7947863676826681, "relative_weight_rmse": 0.014203867204328913 }, { "index": 193, "total": 252, "projection": "layers.33.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 1.171700695902796, "retained_delta_l2": 1.1623095407540411, "delta_retention_norm_ratio": 0.9919850221292913, "delta_cosine": 0.827751229147002, "relative_weight_rmse": 0.01555032560869696 }, { "index": 194, "total": 252, "projection": "layers.33.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1288, "parameters": 16777216, "reference_delta_l2": 1.9538396201038633, "retained_delta_l2": 1.8052516114707444, "delta_retention_norm_ratio": 0.9239507648917366, "delta_cosine": 0.7410008753100303, "relative_weight_rmse": 0.012285579486915223 }, { "index": 195, "total": 252, "projection": "layers.33.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1299, "parameters": 16777216, "reference_delta_l2": 1.6938910069946165, "retained_delta_l2": 1.5612559026371533, "delta_retention_norm_ratio": 0.9216979700525179, "delta_cosine": 0.7425285502146433, "relative_weight_rmse": 0.011916875261304336 }, { "index": 196, "total": 252, "projection": "layers.33.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 1.2353249119776444, "retained_delta_l2": 1.186066026793788, "delta_retention_norm_ratio": 0.9601247536528691, "delta_cosine": 0.7866217684692136, "relative_weight_rmse": 0.012733576318062864 }, { "index": 197, "total": 252, "projection": "layers.34.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4203, "parameters": 50331648, "reference_delta_l2": 3.250888771198972, "retained_delta_l2": 3.0194243808532835, "delta_retention_norm_ratio": 0.9287996586052616, "delta_cosine": 0.7451810336261245, "relative_weight_rmse": 0.011829284789644647 }, { "index": 198, "total": 252, "projection": "layers.34.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3786, "parameters": 50331648, "reference_delta_l2": 3.5072204020963986, "retained_delta_l2": 3.3023841588143523, "delta_retention_norm_ratio": 0.9415958451999173, "delta_cosine": 0.7566511228221033, "relative_weight_rmse": 0.013095487324174874 }, { "index": 199, "total": 252, "projection": "layers.34.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3817, "parameters": 50331648, "reference_delta_l2": 3.1199293402159056, "retained_delta_l2": 2.7957443046500536, "delta_retention_norm_ratio": 0.896092186644388, "delta_cosine": 0.7096322687115229, "relative_weight_rmse": 0.011414938266865215 }, { "index": 200, "total": 252, "projection": "layers.34.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0316, "parameters": 4194304, "reference_delta_l2": 0.788029415327778, "retained_delta_l2": 0.7265886893910003, "delta_retention_norm_ratio": 0.9220324460715446, "delta_cosine": 0.7525129228802909, "relative_weight_rmse": 0.011620429007118772 }, { "index": 201, "total": 252, "projection": "layers.34.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1281, "parameters": 16777216, "reference_delta_l2": 2.3613394109958232, "retained_delta_l2": 2.289951534807643, "delta_retention_norm_ratio": 0.9697680579692377, "delta_cosine": 0.7925439024667102, "relative_weight_rmse": 0.01402741762220263 }, { "index": 202, "total": 252, "projection": "layers.34.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1288, "parameters": 16777216, "reference_delta_l2": 2.785871038243405, "retained_delta_l2": 2.7988799538483624, "delta_retention_norm_ratio": 1.004669604380955, "delta_cosine": 0.8594569487442454, "relative_weight_rmse": 0.015266239440760419 }, { "index": 203, "total": 252, "projection": "layers.34.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 0.9814467233633697, "retained_delta_l2": 0.8768093993968407, "delta_retention_norm_ratio": 0.8933846112319351, "delta_cosine": 0.7153985705871384, "relative_weight_rmse": 0.011405559396125359 }, { "index": 204, "total": 252, "projection": "layers.35.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4213, "parameters": 50331648, "reference_delta_l2": 2.3183374458505197, "retained_delta_l2": 2.0695365530536636, "delta_retention_norm_ratio": 0.8926813293542867, "delta_cosine": 0.7130042803163568, "relative_weight_rmse": 0.00977575248991696 }, { "index": 205, "total": 252, "projection": "layers.35.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3849, "parameters": 50331648, "reference_delta_l2": 2.614786567527581, "retained_delta_l2": 2.279114457992027, "delta_retention_norm_ratio": 0.871625426830554, "delta_cosine": 0.686332682020759, "relative_weight_rmse": 0.010501353670140995 }, { "index": 206, "total": 252, "projection": "layers.35.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3791, "parameters": 50331648, "reference_delta_l2": 2.6518225242333027, "retained_delta_l2": 2.27767724625173, "delta_retention_norm_ratio": 0.8589101364957498, "delta_cosine": 0.6736036409544919, "relative_weight_rmse": 0.010318953190741935 }, { "index": 207, "total": 252, "projection": "layers.35.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0309, "parameters": 4194304, "reference_delta_l2": 0.8334287809981532, "retained_delta_l2": 0.7635522219713372, "delta_retention_norm_ratio": 0.9161577322262274, "delta_cosine": 0.73447000352539, "relative_weight_rmse": 0.012233826186733552 }, { "index": 208, "total": 252, "projection": "layers.35.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.13, "parameters": 16777216, "reference_delta_l2": 1.9820359827690113, "retained_delta_l2": 1.8810726256763255, "delta_retention_norm_ratio": 0.9490607849855306, "delta_cosine": 0.7679138582696287, "relative_weight_rmse": 0.013001248772620494 }, { "index": 209, "total": 252, "projection": "layers.35.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1264, "parameters": 16777216, "reference_delta_l2": 1.9167783776293494, "retained_delta_l2": 1.8284949611298134, "delta_retention_norm_ratio": 0.9539417715006134, "delta_cosine": 0.7741266142645553, "relative_weight_rmse": 0.01306133912536974 }, { "index": 210, "total": 252, "projection": "layers.35.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 0.9045634157104502, "retained_delta_l2": 0.7983614933900623, "delta_retention_norm_ratio": 0.8825931709420548, "delta_cosine": 0.6994521063502559, "relative_weight_rmse": 0.011420838616268927 }, { "index": 211, "total": 252, "projection": "layers.4.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4191, "parameters": 50331648, "reference_delta_l2": 2.221336647346869, "retained_delta_l2": 1.9045813778894223, "delta_retention_norm_ratio": 0.8574033027205603, "delta_cosine": 0.6727181068599302, "relative_weight_rmse": 0.010822728427872375 }, { "index": 212, "total": 252, "projection": "layers.4.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.381, "parameters": 50331648, "reference_delta_l2": 3.309274307029437, "retained_delta_l2": 2.9183820142372894, "delta_retention_norm_ratio": 0.8818797547360071, "delta_cosine": 0.7088330041186263, "relative_weight_rmse": 0.010818316989641936 }, { "index": 213, "total": 252, "projection": "layers.4.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3791, "parameters": 50331648, "reference_delta_l2": 3.3002149367491143, "retained_delta_l2": 3.1780020825172097, "delta_retention_norm_ratio": 0.9629682137151072, "delta_cosine": 0.7953073969949832, "relative_weight_rmse": 0.014014920542321037 }, { "index": 214, "total": 252, "projection": "layers.4.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0321, "parameters": 4194304, "reference_delta_l2": 0.5923128812842459, "retained_delta_l2": 0.44706547078012393, "delta_retention_norm_ratio": 0.7547792474322047, "delta_cosine": 0.5777734174137593, "relative_weight_rmse": 0.00945310317345178 }, { "index": 215, "total": 252, "projection": "layers.4.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1283, "parameters": 16777216, "reference_delta_l2": 1.6233424860329353, "retained_delta_l2": 1.4087840092320407, "delta_retention_norm_ratio": 0.8678291989232508, "delta_cosine": 0.6799719691456307, "relative_weight_rmse": 0.011781348708998187 }, { "index": 216, "total": 252, "projection": "layers.4.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1281, "parameters": 16777216, "reference_delta_l2": 1.2599063228139533, "retained_delta_l2": 0.9953410590893825, "delta_retention_norm_ratio": 0.7900119564971512, "delta_cosine": 0.607803419298993, "relative_weight_rmse": 0.010232583667853115 }, { "index": 217, "total": 252, "projection": "layers.4.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0301, "parameters": 4194304, "reference_delta_l2": 1.1362098663278084, "retained_delta_l2": 1.0785328669695842, "delta_retention_norm_ratio": 0.949237371486102, "delta_cosine": 0.7692033108834336, "relative_weight_rmse": 0.013542298508869052 }, { "index": 218, "total": 252, "projection": "layers.5.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4233, "parameters": 50331648, "reference_delta_l2": 1.803962899947611, "retained_delta_l2": 1.342634588105705, "delta_retention_norm_ratio": 0.744269512496458, "delta_cosine": 0.5694831159509246, "relative_weight_rmse": 0.009042785321309456 }, { "index": 219, "total": 252, "projection": "layers.5.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3847, "parameters": 50331648, "reference_delta_l2": 2.9502308395883783, "retained_delta_l2": 2.4860241230127524, "delta_retention_norm_ratio": 0.8426541034191098, "delta_cosine": 0.66015731037967, "relative_weight_rmse": 0.009986779270096871 }, { "index": 220, "total": 252, "projection": "layers.5.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3806, "parameters": 50331648, "reference_delta_l2": 3.173490918626583, "retained_delta_l2": 2.9589975451196926, "delta_retention_norm_ratio": 0.9324109067878731, "delta_cosine": 0.7491525974044665, "relative_weight_rmse": 0.013117147848501721 }, { "index": 221, "total": 252, "projection": "layers.5.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.6193610286474956, "retained_delta_l2": 0.487339283051826, "delta_retention_norm_ratio": 0.7868420202608376, "delta_cosine": 0.6077146773250078, "relative_weight_rmse": 0.009850471449688411 }, { "index": 222, "total": 252, "projection": "layers.5.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1345, "parameters": 16777216, "reference_delta_l2": 1.4660458596862542, "retained_delta_l2": 1.2308176050578241, "delta_retention_norm_ratio": 0.8395491838988101, "delta_cosine": 0.6516145241740764, "relative_weight_rmse": 0.011341592423666287 }, { "index": 223, "total": 252, "projection": "layers.5.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1283, "parameters": 16777216, "reference_delta_l2": 1.4671626671458942, "retained_delta_l2": 1.252449986018564, "delta_retention_norm_ratio": 0.8536544815817759, "delta_cosine": 0.6701923660259413, "relative_weight_rmse": 0.011180446979722173 }, { "index": 224, "total": 252, "projection": "layers.5.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0296, "parameters": 4194304, "reference_delta_l2": 1.044090832035962, "retained_delta_l2": 0.9792640619326973, "delta_retention_norm_ratio": 0.9379107946222902, "delta_cosine": 0.760232127059494, "relative_weight_rmse": 0.012863542624411998 }, { "index": 225, "total": 252, "projection": "layers.6.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4213, "parameters": 50331648, "reference_delta_l2": 4.766108258771339, "retained_delta_l2": 4.77873869719586, "delta_retention_norm_ratio": 1.0026500527765556, "delta_cosine": 0.8442033313899234, "relative_weight_rmse": 0.01543176975270308 }, { "index": 226, "total": 252, "projection": "layers.6.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3839, "parameters": 50331648, "reference_delta_l2": 3.1029703699387263, "retained_delta_l2": 2.712238549129781, "delta_retention_norm_ratio": 0.874078133457439, "delta_cosine": 0.6918195489369741, "relative_weight_rmse": 0.010674646842871805 }, { "index": 227, "total": 252, "projection": "layers.6.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3821, "parameters": 50331648, "reference_delta_l2": 3.3821355337004557, "retained_delta_l2": 3.1934349534654216, "delta_retention_norm_ratio": 0.9442066770078333, "delta_cosine": 0.7628082158548107, "relative_weight_rmse": 0.013209469970077926 }, { "index": 228, "total": 252, "projection": "layers.6.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0315, "parameters": 4194304, "reference_delta_l2": 0.5910609699867302, "retained_delta_l2": 0.4457745114424727, "delta_retention_norm_ratio": 0.7541937872373483, "delta_cosine": 0.5775529443999605, "relative_weight_rmse": 0.009558571249523508 }, { "index": 229, "total": 252, "projection": "layers.6.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1303, "parameters": 16777216, "reference_delta_l2": 1.3625651734345294, "retained_delta_l2": 1.1040539958059539, "delta_retention_norm_ratio": 0.8102761007923289, "delta_cosine": 0.626134758602679, "relative_weight_rmse": 0.01038159614631551 }, { "index": 230, "total": 252, "projection": "layers.6.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1292, "parameters": 16777216, "reference_delta_l2": 1.4025439238449944, "retained_delta_l2": 1.1632560404983912, "delta_retention_norm_ratio": 0.8293900966105867, "delta_cosine": 0.6446595803979036, "relative_weight_rmse": 0.010718242315551555 }, { "index": 231, "total": 252, "projection": "layers.6.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.03, "parameters": 4194304, "reference_delta_l2": 0.9850880555955308, "retained_delta_l2": 0.9089428205266216, "delta_retention_norm_ratio": 0.9227021029882695, "delta_cosine": 0.7391486772354445, "relative_weight_rmse": 0.012392920894060367 }, { "index": 232, "total": 252, "projection": "layers.7.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4368, "parameters": 50331648, "reference_delta_l2": 1.8440352305241692, "retained_delta_l2": 1.3640679381239535, "delta_retention_norm_ratio": 0.7397190224702028, "delta_cosine": 0.5653753919522522, "relative_weight_rmse": 0.008955389570752136 }, { "index": 233, "total": 252, "projection": "layers.7.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.4066, "parameters": 50331648, "reference_delta_l2": 2.893335151708075, "retained_delta_l2": 2.477994315015933, "delta_retention_norm_ratio": 0.8564491097939531, "delta_cosine": 0.673437015493994, "relative_weight_rmse": 0.010343431056423151 }, { "index": 234, "total": 252, "projection": "layers.7.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3857, "parameters": 50331648, "reference_delta_l2": 2.967222221507067, "retained_delta_l2": 2.682568633972963, "delta_retention_norm_ratio": 0.9040673174153006, "delta_cosine": 0.7157974740017237, "relative_weight_rmse": 0.012361975496102632 }, { "index": 235, "total": 252, "projection": "layers.7.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0318, "parameters": 4194304, "reference_delta_l2": 0.7208766702180713, "retained_delta_l2": 0.6114166942831658, "delta_retention_norm_ratio": 0.8481571391375546, "delta_cosine": 0.6619191498041295, "relative_weight_rmse": 0.01075427974342996 }, { "index": 236, "total": 252, "projection": "layers.7.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1302, "parameters": 16777216, "reference_delta_l2": 1.3658873172509274, "retained_delta_l2": 1.106406551160283, "delta_retention_norm_ratio": 0.8100276920259483, "delta_cosine": 0.6252429909572943, "relative_weight_rmse": 0.010549164315745227 }, { "index": 237, "total": 252, "projection": "layers.7.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1301, "parameters": 16777216, "reference_delta_l2": 1.6847457418984, "retained_delta_l2": 1.5226516861692236, "delta_retention_norm_ratio": 0.9037872293141835, "delta_cosine": 0.7177071108487745, "relative_weight_rmse": 0.012113206962301877 }, { "index": 238, "total": 252, "projection": "layers.7.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 0.9868820964566372, "retained_delta_l2": 0.8897068211702872, "delta_retention_norm_ratio": 0.9015330446917073, "delta_cosine": 0.7170913768002884, "relative_weight_rmse": 0.01196194767053509 }, { "index": 239, "total": 252, "projection": "layers.8.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4308, "parameters": 50331648, "reference_delta_l2": 2.6355924662208907, "retained_delta_l2": 2.2731708175513594, "delta_retention_norm_ratio": 0.8624894958858346, "delta_cosine": 0.6768140251766139, "relative_weight_rmse": 0.011039891760100158 }, { "index": 240, "total": 252, "projection": "layers.8.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.389, "parameters": 50331648, "reference_delta_l2": 2.202283141318376, "retained_delta_l2": 1.659652546325219, "delta_retention_norm_ratio": 0.7536054357350632, "delta_cosine": 0.5770296667376316, "relative_weight_rmse": 0.009465511365822644 }, { "index": 241, "total": 252, "projection": "layers.8.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3955, "parameters": 50331648, "reference_delta_l2": 2.6806339499510345, "retained_delta_l2": 2.2858577300024723, "delta_retention_norm_ratio": 0.8527302767482395, "delta_cosine": 0.666012255909814, "relative_weight_rmse": 0.011428825233552205 }, { "index": 242, "total": 252, "projection": "layers.8.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0348, "parameters": 4194304, "reference_delta_l2": 0.892108098567984, "retained_delta_l2": 0.8088306720344752, "delta_retention_norm_ratio": 0.9066509690168869, "delta_cosine": 0.7194814890534805, "relative_weight_rmse": 0.0121186655265901 }, { "index": 243, "total": 252, "projection": "layers.8.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1428, "parameters": 16777216, "reference_delta_l2": 1.5711709173138488, "retained_delta_l2": 1.3450190219279747, "delta_retention_norm_ratio": 0.8560615570885729, "delta_cosine": 0.6701946040661244, "relative_weight_rmse": 0.01130083083234018 }, { "index": 244, "total": 252, "projection": "layers.8.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.147, "parameters": 16777216, "reference_delta_l2": 1.5935161523801298, "retained_delta_l2": 1.382567306616549, "delta_retention_norm_ratio": 0.8676205161469493, "delta_cosine": 0.6790418645088696, "relative_weight_rmse": 0.011833231993008326 }, { "index": 245, "total": 252, "projection": "layers.8.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.035, "parameters": 4194304, "reference_delta_l2": 0.7958179709276099, "retained_delta_l2": 0.6599420021002363, "delta_retention_norm_ratio": 0.8292625024928807, "delta_cosine": 0.6433285511923893, "relative_weight_rmse": 0.01062109515860514 }, { "index": 246, "total": 252, "projection": "layers.9.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4556, "parameters": 50331648, "reference_delta_l2": 2.4632649867432574, "retained_delta_l2": 2.0916648368795094, "delta_retention_norm_ratio": 0.8491432501725892, "delta_cosine": 0.6636211036150158, "relative_weight_rmse": 0.010863964650101629 }, { "index": 247, "total": 252, "projection": "layers.9.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.4064, "parameters": 50331648, "reference_delta_l2": 2.6912322423445403, "retained_delta_l2": 2.2009723269453074, "delta_retention_norm_ratio": 0.8178306919464781, "delta_cosine": 0.634134962968425, "relative_weight_rmse": 0.010350243159792981 }, { "index": 248, "total": 252, "projection": "layers.9.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.4039, "parameters": 50331648, "reference_delta_l2": 3.019301808316451, "retained_delta_l2": 2.736703036249453, "delta_retention_norm_ratio": 0.9064026089446905, "delta_cosine": 0.7213458216286287, "relative_weight_rmse": 0.01237028304932083 }, { "index": 249, "total": 252, "projection": "layers.9.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0322, "parameters": 4194304, "reference_delta_l2": 0.8372366151629234, "retained_delta_l2": 0.7554391472146549, "delta_retention_norm_ratio": 0.9023006561504109, "delta_cosine": 0.7181386352936268, "relative_weight_rmse": 0.01177065462703115 }, { "index": 250, "total": 252, "projection": "layers.9.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1401, "parameters": 16777216, "reference_delta_l2": 1.5248485846368605, "retained_delta_l2": 1.2996361778554804, "delta_retention_norm_ratio": 0.8523050688111345, "delta_cosine": 0.6667510008390406, "relative_weight_rmse": 0.011153659988921252 }, { "index": 251, "total": 252, "projection": "layers.9.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1403, "parameters": 16777216, "reference_delta_l2": 1.803779278376794, "retained_delta_l2": 1.6567698226437095, "delta_retention_norm_ratio": 0.918499199156242, "delta_cosine": 0.7350062150712456, "relative_weight_rmse": 0.0126925054191312 }, { "index": 252, "total": 252, "projection": "layers.9.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0309, "parameters": 4194304, "reference_delta_l2": 0.7978971622820221, "retained_delta_l2": 0.6671771373215317, "delta_retention_norm_ratio": 0.8361693321647802, "delta_cosine": 0.6525337153106847, "relative_weight_rmse": 0.010395554903376797 } ] }