{ "schema_version": 1, "format": "comfy_scaled_fp8_merged_text_encoder", "scale_policy": "preserve_stock_per_tensor_scale", "base": "/workspace/Ideogram/checkpoints/text_encoders/qwen3vl_8b_fp8_scaled.safetensors", "base_sha256": "4ba424cf62e51392e4d1a39933e803706f4e823c1065f36aaf149c6453f66bcd", "adapter": "/workspace/Ideogram/artifacts/phase2_balanced_4000_8state_debanner_s020_v1_checkpoints/step_00002540/adapters.safetensors", "adapter_sha256": "f8fe52d0b625e933521e5d9346da158bd5462738c1af4ea3a046397027f25569", "adapter_global_step": "2540", "output": "/workspace/Ideogram/artifacts/huggingface_release/s020_v1/qwen3vl_8b_ideogram4_nl_s020_v1_step_00002540_fp8_scaled.safetensors", "output_bytes": 10588637800, "output_sha256": "4b164ff3c6ef27af89cb14c1bdb5b6e6d20c72bf258308b5ee77b811f976dcf7", "output_tensors": 1254, "merged_projections": 252, "lora_rank": 64, "lora_alpha": 64.0, "elapsed_seconds": 64.148, "projections": [ { "index": 1, "total": 252, "projection": "layers.0.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4236, "parameters": 50331648, "reference_delta_l2": 3.126840741732905, "retained_delta_l2": 2.847031594462536, "delta_retention_norm_ratio": 0.9105137835976584, "delta_cosine": 0.725848228993972, "relative_weight_rmse": 0.012307521162095323 }, { "index": 2, "total": 252, "projection": "layers.0.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3782, "parameters": 50331648, "reference_delta_l2": 3.3155131653878125, "retained_delta_l2": 3.0297597793802833, "delta_retention_norm_ratio": 0.9138132253581008, "delta_cosine": 0.7264097571772404, "relative_weight_rmse": 0.013511860068981345 }, { "index": 3, "total": 252, "projection": "layers.0.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.376, "parameters": 50331648, "reference_delta_l2": 2.93946837412769, "retained_delta_l2": 2.629080052603913, "delta_retention_norm_ratio": 0.8944066470468874, "delta_cosine": 0.7074603179950618, "relative_weight_rmse": 0.013038171006044602 }, { "index": 4, "total": 252, "projection": "layers.0.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 0.6028464680738916, "retained_delta_l2": 0.4418626005952603, "delta_retention_norm_ratio": 0.7329604202659119, "delta_cosine": 0.5606693910696365, "relative_weight_rmse": 0.009013772200868762 }, { "index": 5, "total": 252, "projection": "layers.0.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1262, "parameters": 16777216, "reference_delta_l2": 1.973553111162272, "retained_delta_l2": 1.861623920055665, "delta_retention_norm_ratio": 0.9432854426498362, "delta_cosine": 0.7628375185005537, "relative_weight_rmse": 0.01362450712127664 }, { "index": 6, "total": 252, "projection": "layers.0.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1366, "parameters": 16777216, "reference_delta_l2": 1.225185293772824, "retained_delta_l2": 0.9533197670271886, "delta_retention_norm_ratio": 0.7781025220206036, "delta_cosine": 0.5987807928868673, "relative_weight_rmse": 0.009804670894225527 }, { "index": 7, "total": 252, "projection": "layers.0.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0293, "parameters": 4194304, "reference_delta_l2": 0.7953509807620676, "retained_delta_l2": 0.6926766334641751, "delta_retention_norm_ratio": 0.8709068703234454, "delta_cosine": 0.6838780163989387, "relative_weight_rmse": 0.01198416383607124 }, { "index": 8, "total": 252, "projection": "layers.1.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4275, "parameters": 50331648, "reference_delta_l2": 2.0361246515843954, "retained_delta_l2": 1.900496503757514, "delta_retention_norm_ratio": 0.933389074327378, "delta_cosine": 0.756480270633347, "relative_weight_rmse": 0.011983444253769262 }, { "index": 9, "total": 252, "projection": "layers.1.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3845, "parameters": 50331648, "reference_delta_l2": 3.827864663760918, "retained_delta_l2": 3.7945581449684007, "delta_retention_norm_ratio": 0.9912989298948219, "delta_cosine": 0.8456527848261863, "relative_weight_rmse": 0.014356902032464577 }, { "index": 10, "total": 252, "projection": "layers.1.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3768, "parameters": 50331648, "reference_delta_l2": 4.228079577870509, "retained_delta_l2": 4.225241074129967, "delta_retention_norm_ratio": 0.9993286541352253, "delta_cosine": 0.9294420997507352, "relative_weight_rmse": 0.01576108016664263 }, { "index": 11, "total": 252, "projection": "layers.1.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 0.6761678837629111, "retained_delta_l2": 0.5551086850164083, "delta_retention_norm_ratio": 0.8209628087143066, "delta_cosine": 0.6372181263878011, "relative_weight_rmse": 0.010816268701898937 }, { "index": 12, "total": 252, "projection": "layers.1.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1266, "parameters": 16777216, "reference_delta_l2": 1.7122907077208531, "retained_delta_l2": 1.5313364953859028, "delta_retention_norm_ratio": 0.8943203910883745, "delta_cosine": 0.7050326938351207, "relative_weight_rmse": 0.012655305524148125 }, { "index": 13, "total": 252, "projection": "layers.1.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1263, "parameters": 16777216, "reference_delta_l2": 2.0928366814121127, "retained_delta_l2": 2.0236042080333014, "delta_retention_norm_ratio": 0.9669193138701594, "delta_cosine": 0.7927232891654293, "relative_weight_rmse": 0.014637441165904277 }, { "index": 14, "total": 252, "projection": "layers.1.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.029, "parameters": 4194304, "reference_delta_l2": 0.9145859087024274, "retained_delta_l2": 0.8318342758362796, "delta_retention_norm_ratio": 0.9095201095066595, "delta_cosine": 0.7222397921654805, "relative_weight_rmse": 0.01243292081589284 }, { "index": 15, "total": 252, "projection": "layers.10.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4172, "parameters": 50331648, "reference_delta_l2": 2.81236752270305, "retained_delta_l2": 2.493766399212893, "delta_retention_norm_ratio": 0.8867142644344223, "delta_cosine": 0.7007063524167806, "relative_weight_rmse": 0.01166425786353078 }, { "index": 16, "total": 252, "projection": "layers.10.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3757, "parameters": 50331648, "reference_delta_l2": 2.3905832278522254, "retained_delta_l2": 1.8532254458493804, "delta_retention_norm_ratio": 0.7752189609036854, "delta_cosine": 0.5960260535930171, "relative_weight_rmse": 0.009653432972177991 }, { "index": 17, "total": 252, "projection": "layers.10.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3774, "parameters": 50331648, "reference_delta_l2": 2.8556621352623095, "retained_delta_l2": 2.5270634634012414, "delta_retention_norm_ratio": 0.8849308299453695, "delta_cosine": 0.6974295939624287, "relative_weight_rmse": 0.011906604700482923 }, { "index": 18, "total": 252, "projection": "layers.10.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.03, "parameters": 4194304, "reference_delta_l2": 0.897891377626497, "retained_delta_l2": 0.8251920079584362, "delta_retention_norm_ratio": 0.9190332244193773, "delta_cosine": 0.7371071941855177, "relative_weight_rmse": 0.011969155540596983 }, { "index": 19, "total": 252, "projection": "layers.10.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1257, "parameters": 16777216, "reference_delta_l2": 1.7752074338164896, "retained_delta_l2": 1.586539641118029, "delta_retention_norm_ratio": 0.893720706040056, "delta_cosine": 0.7069774975623085, "relative_weight_rmse": 0.012172454143964525 }, { "index": 20, "total": 252, "projection": "layers.10.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1253, "parameters": 16777216, "reference_delta_l2": 2.1185128595130265, "retained_delta_l2": 2.020718756582176, "delta_retention_norm_ratio": 0.9538383246097785, "delta_cosine": 0.7798847138295467, "relative_weight_rmse": 0.01345056123715535 }, { "index": 21, "total": 252, "projection": "layers.10.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.029, "parameters": 4194304, "reference_delta_l2": 0.9100838136534896, "retained_delta_l2": 0.8128102757212403, "delta_retention_norm_ratio": 0.8931158466144463, "delta_cosine": 0.7113298638535298, "relative_weight_rmse": 0.011205869215500954 }, { "index": 22, "total": 252, "projection": "layers.11.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4191, "parameters": 50331648, "reference_delta_l2": 2.6097737594933634, "retained_delta_l2": 2.2340911828455416, "delta_retention_norm_ratio": 0.8560478373724039, "delta_cosine": 0.6706686430394193, "relative_weight_rmse": 0.011050285955504799 }, { "index": 23, "total": 252, "projection": "layers.11.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.378, "parameters": 50331648, "reference_delta_l2": 2.7751145691860644, "retained_delta_l2": 2.343027160291451, "delta_retention_norm_ratio": 0.8442992539146433, "delta_cosine": 0.6596664167663548, "relative_weight_rmse": 0.010816182538165395 }, { "index": 24, "total": 252, "projection": "layers.11.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3777, "parameters": 50331648, "reference_delta_l2": 2.675125117521029, "retained_delta_l2": 2.2732189655888804, "delta_retention_norm_ratio": 0.8497617366381035, "delta_cosine": 0.6633901224047348, "relative_weight_rmse": 0.011374993862828652 }, { "index": 25, "total": 252, "projection": "layers.11.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0303, "parameters": 4194304, "reference_delta_l2": 1.129744485973673, "retained_delta_l2": 1.0869780166591216, "delta_retention_norm_ratio": 0.9621450072600329, "delta_cosine": 0.7872812397310865, "relative_weight_rmse": 0.013666281685033765 }, { "index": 26, "total": 252, "projection": "layers.11.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1274, "parameters": 16777216, "reference_delta_l2": 1.670846789488529, "retained_delta_l2": 1.4674471513825977, "delta_retention_norm_ratio": 0.8782655361427874, "delta_cosine": 0.6931815425436411, "relative_weight_rmse": 0.011730107850665127 }, { "index": 27, "total": 252, "projection": "layers.11.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.126, "parameters": 16777216, "reference_delta_l2": 1.4076682377658807, "retained_delta_l2": 1.1708727712056912, "delta_retention_norm_ratio": 0.8317817649022123, "delta_cosine": 0.6466330501407813, "relative_weight_rmse": 0.010758078465344743 }, { "index": 28, "total": 252, "projection": "layers.11.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0292, "parameters": 4194304, "reference_delta_l2": 0.9977954436103857, "retained_delta_l2": 0.9075995021759654, "delta_retention_norm_ratio": 0.9096047771995645, "delta_cosine": 0.7270939597590178, "relative_weight_rmse": 0.01188627268952539 }, { "index": 29, "total": 252, "projection": "layers.12.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4254, "parameters": 50331648, "reference_delta_l2": 2.56819137012028, "retained_delta_l2": 2.188648995444725, "delta_retention_norm_ratio": 0.8522141382876078, "delta_cosine": 0.6668181723119647, "relative_weight_rmse": 0.01083587743189139 }, { "index": 30, "total": 252, "projection": "layers.12.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3844, "parameters": 50331648, "reference_delta_l2": 3.215622881801083, "retained_delta_l2": 2.8836738663452626, "delta_retention_norm_ratio": 0.8967699174755547, "delta_cosine": 0.7129079018317985, "relative_weight_rmse": 0.01198643496827087 }, { "index": 31, "total": 252, "projection": "layers.12.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3805, "parameters": 50331648, "reference_delta_l2": 2.6879330439167055, "retained_delta_l2": 2.291630979320779, "delta_retention_norm_ratio": 0.8525625236488567, "delta_cosine": 0.6663007010584944, "relative_weight_rmse": 0.01122343069789282 }, { "index": 32, "total": 252, "projection": "layers.12.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 0.9326365058053449, "retained_delta_l2": 0.8659338721920582, "delta_retention_norm_ratio": 0.9284794952823683, "delta_cosine": 0.7478119556266251, "relative_weight_rmse": 0.01229225064365473 }, { "index": 33, "total": 252, "projection": "layers.12.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1317, "parameters": 16777216, "reference_delta_l2": 1.7740352657937566, "retained_delta_l2": 1.6069883409975243, "delta_retention_norm_ratio": 0.9058378781881258, "delta_cosine": 0.7219368480569792, "relative_weight_rmse": 0.012034122226082595 }, { "index": 34, "total": 252, "projection": "layers.12.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1301, "parameters": 16777216, "reference_delta_l2": 1.8069565591399073, "retained_delta_l2": 1.6521463119058284, "delta_retention_norm_ratio": 0.9143254183665782, "delta_cosine": 0.7313478948888776, "relative_weight_rmse": 0.012435789153691534 }, { "index": 35, "total": 252, "projection": "layers.12.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0296, "parameters": 4194304, "reference_delta_l2": 1.0869529155925972, "retained_delta_l2": 1.0238572118353657, "delta_retention_norm_ratio": 0.9419517599593242, "delta_cosine": 0.7659206070161154, "relative_weight_rmse": 0.012192291258131012 }, { "index": 36, "total": 252, "projection": "layers.13.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4309, "parameters": 50331648, "reference_delta_l2": 2.345717959028678, "retained_delta_l2": 1.9211998853355021, "delta_retention_norm_ratio": 0.8190242471141068, "delta_cosine": 0.6358526600195066, "relative_weight_rmse": 0.010194365616716888 }, { "index": 37, "total": 252, "projection": "layers.13.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3975, "parameters": 50331648, "reference_delta_l2": 2.9930721247000602, "retained_delta_l2": 2.6193255999400242, "delta_retention_norm_ratio": 0.8751294625760181, "delta_cosine": 0.6879834774883976, "relative_weight_rmse": 0.01189494452769944 }, { "index": 38, "total": 252, "projection": "layers.13.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.4029, "parameters": 50331648, "reference_delta_l2": 2.4019665807886206, "retained_delta_l2": 1.9420047828308697, "delta_retention_norm_ratio": 0.8085061625600407, "delta_cosine": 0.6251286647554096, "relative_weight_rmse": 0.010350310001921261 }, { "index": 39, "total": 252, "projection": "layers.13.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0328, "parameters": 4194304, "reference_delta_l2": 1.341017407179969, "retained_delta_l2": 1.3266552896841128, "delta_retention_norm_ratio": 0.9892901334323031, "delta_cosine": 0.8343821488449628, "relative_weight_rmse": 0.014778051436635804 }, { "index": 40, "total": 252, "projection": "layers.13.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1345, "parameters": 16777216, "reference_delta_l2": 1.7312984063035275, "retained_delta_l2": 1.5520233983981218, "delta_retention_norm_ratio": 0.896450544139197, "delta_cosine": 0.7121618169131606, "relative_weight_rmse": 0.012267247947723397 }, { "index": 41, "total": 252, "projection": "layers.13.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1332, "parameters": 16777216, "reference_delta_l2": 1.4095608507370043, "retained_delta_l2": 1.1734218382297974, "delta_retention_norm_ratio": 0.832473346302333, "delta_cosine": 0.6470345038595087, "relative_weight_rmse": 0.01074976588373817 }, { "index": 42, "total": 252, "projection": "layers.13.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 1.02837419663425, "retained_delta_l2": 0.9472396712269158, "delta_retention_norm_ratio": 0.9211040828592569, "delta_cosine": 0.7384602764192114, "relative_weight_rmse": 0.012317781541514026 }, { "index": 43, "total": 252, "projection": "layers.14.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4305, "parameters": 50331648, "reference_delta_l2": 2.4695213210201166, "retained_delta_l2": 2.088833734616667, "delta_retention_norm_ratio": 0.8458455964064348, "delta_cosine": 0.6615175643703667, "relative_weight_rmse": 0.010523678881525165 }, { "index": 44, "total": 252, "projection": "layers.14.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3935, "parameters": 50331648, "reference_delta_l2": 2.73564026062679, "retained_delta_l2": 2.3210056632930196, "delta_retention_norm_ratio": 0.8484323383810819, "delta_cosine": 0.6617345073961669, "relative_weight_rmse": 0.011367344684873312 }, { "index": 45, "total": 252, "projection": "layers.14.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3944, "parameters": 50331648, "reference_delta_l2": 2.9456908148003613, "retained_delta_l2": 2.611415626037734, "delta_retention_norm_ratio": 0.8865206127258531, "delta_cosine": 0.7009710931147657, "relative_weight_rmse": 0.011786178342384171 }, { "index": 46, "total": 252, "projection": "layers.14.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 0.7290877266076532, "retained_delta_l2": 0.6191256185610067, "delta_retention_norm_ratio": 0.8491784952158153, "delta_cosine": 0.666182717165303, "relative_weight_rmse": 0.01072129480320891 }, { "index": 47, "total": 252, "projection": "layers.14.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1339, "parameters": 16777216, "reference_delta_l2": 1.5562308772486637, "retained_delta_l2": 1.3548889172329652, "delta_retention_norm_ratio": 0.8706220503916099, "delta_cosine": 0.6849544987072025, "relative_weight_rmse": 0.011444515120076961 }, { "index": 48, "total": 252, "projection": "layers.14.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1336, "parameters": 16777216, "reference_delta_l2": 2.3181710795261905, "retained_delta_l2": 2.259554226479229, "delta_retention_norm_ratio": 0.9747141815525789, "delta_cosine": 0.8054773380596841, "relative_weight_rmse": 0.013913795311187202 }, { "index": 49, "total": 252, "projection": "layers.14.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.03, "parameters": 4194304, "reference_delta_l2": 1.0169140246366664, "retained_delta_l2": 0.9438884227724618, "delta_retention_norm_ratio": 0.928189010973375, "delta_cosine": 0.7456435016623865, "relative_weight_rmse": 0.012379671180818227 }, { "index": 50, "total": 252, "projection": "layers.15.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4343, "parameters": 50331648, "reference_delta_l2": 2.37291388097089, "retained_delta_l2": 1.9747544435249724, "delta_retention_norm_ratio": 0.8322065370181033, "delta_cosine": 0.6483283407698353, "relative_weight_rmse": 0.010305350636473356 }, { "index": 51, "total": 252, "projection": "layers.15.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3924, "parameters": 50331648, "reference_delta_l2": 2.877735534294013, "retained_delta_l2": 2.5181948245020824, "delta_retention_norm_ratio": 0.875061239816071, "delta_cosine": 0.6869561824692126, "relative_weight_rmse": 0.01203124922287954 }, { "index": 52, "total": 252, "projection": "layers.15.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3941, "parameters": 50331648, "reference_delta_l2": 2.7887238690235363, "retained_delta_l2": 2.4400914015325093, "delta_retention_norm_ratio": 0.8749849451343852, "delta_cosine": 0.6893492075249347, "relative_weight_rmse": 0.011424688533661688 }, { "index": 53, "total": 252, "projection": "layers.15.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0317, "parameters": 4194304, "reference_delta_l2": 0.8136634946472183, "retained_delta_l2": 0.7194351998141398, "delta_retention_norm_ratio": 0.884192549557685, "delta_cosine": 0.6989657732038983, "relative_weight_rmse": 0.01158525789051138 }, { "index": 54, "total": 252, "projection": "layers.15.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1354, "parameters": 16777216, "reference_delta_l2": 2.14888279095195, "retained_delta_l2": 2.0648059108914714, "delta_retention_norm_ratio": 0.9608741433388125, "delta_cosine": 0.7884679794096899, "relative_weight_rmse": 0.01383434690156249 }, { "index": 55, "total": 252, "projection": "layers.15.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1339, "parameters": 16777216, "reference_delta_l2": 2.2363308934404, "retained_delta_l2": 2.1612427983529505, "delta_retention_norm_ratio": 0.9664235309239354, "delta_cosine": 0.8010344733340968, "relative_weight_rmse": 0.013448329113463752 }, { "index": 56, "total": 252, "projection": "layers.15.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0318, "parameters": 4194304, "reference_delta_l2": 0.8579609120298205, "retained_delta_l2": 0.7628252507622412, "delta_retention_norm_ratio": 0.8891142242803333, "delta_cosine": 0.7053941734827593, "relative_weight_rmse": 0.011222617076479648 }, { "index": 57, "total": 252, "projection": "layers.16.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4375, "parameters": 50331648, "reference_delta_l2": 2.4925761772926727, "retained_delta_l2": 2.1300505488925414, "delta_retention_norm_ratio": 0.85455785395739, "delta_cosine": 0.6692738300226725, "relative_weight_rmse": 0.010669613300324836 }, { "index": 58, "total": 252, "projection": "layers.16.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3967, "parameters": 50331648, "reference_delta_l2": 2.639823835206382, "retained_delta_l2": 2.2568373816548224, "delta_retention_norm_ratio": 0.85491969257804, "delta_cosine": 0.6684142838746178, "relative_weight_rmse": 0.011071951086947165 }, { "index": 59, "total": 252, "projection": "layers.16.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3994, "parameters": 50331648, "reference_delta_l2": 2.5574051385465872, "retained_delta_l2": 2.155985075223233, "delta_retention_norm_ratio": 0.8430361864559763, "delta_cosine": 0.657171055105935, "relative_weight_rmse": 0.010889774960224613 }, { "index": 60, "total": 252, "projection": "layers.16.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0311, "parameters": 4194304, "reference_delta_l2": 0.7403652978861993, "retained_delta_l2": 0.6378164203841092, "delta_retention_norm_ratio": 0.8614888112734684, "delta_cosine": 0.6754536654041666, "relative_weight_rmse": 0.011217294766259881 }, { "index": 61, "total": 252, "projection": "layers.16.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1357, "parameters": 16777216, "reference_delta_l2": 1.6664737578904119, "retained_delta_l2": 1.4904171603264238, "delta_retention_norm_ratio": 0.8943538134156651, "delta_cosine": 0.709376769947244, "relative_weight_rmse": 0.011938452222249625 }, { "index": 62, "total": 252, "projection": "layers.16.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.134, "parameters": 16777216, "reference_delta_l2": 1.810967498456618, "retained_delta_l2": 1.6792081046389524, "delta_retention_norm_ratio": 0.9272436452172905, "delta_cosine": 0.7481180718262056, "relative_weight_rmse": 0.012466719594668663 }, { "index": 63, "total": 252, "projection": "layers.16.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 0.9588623100041671, "retained_delta_l2": 0.8756644620899365, "delta_retention_norm_ratio": 0.9132327477613871, "delta_cosine": 0.7301267251224983, "relative_weight_rmse": 0.012151192168392035 }, { "index": 64, "total": 252, "projection": "layers.17.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4348, "parameters": 50331648, "reference_delta_l2": 2.5816971370993724, "retained_delta_l2": 2.2212677742122406, "delta_retention_norm_ratio": 0.8603905323720168, "delta_cosine": 0.6749114085537792, "relative_weight_rmse": 0.010931229419799657 }, { "index": 65, "total": 252, "projection": "layers.17.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3935, "parameters": 50331648, "reference_delta_l2": 3.537919615556736, "retained_delta_l2": 3.3171539258685923, "delta_retention_norm_ratio": 0.9376001397212629, "delta_cosine": 0.7563796909416031, "relative_weight_rmse": 0.013295309785955068 }, { "index": 66, "total": 252, "projection": "layers.17.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3948, "parameters": 50331648, "reference_delta_l2": 3.184727757412606, "retained_delta_l2": 2.900798539966914, "delta_retention_norm_ratio": 0.9108466283233053, "delta_cosine": 0.7254635730513106, "relative_weight_rmse": 0.012378258470841225 }, { "index": 67, "total": 252, "projection": "layers.17.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0327, "parameters": 4194304, "reference_delta_l2": 0.7873391801685035, "retained_delta_l2": 0.7046799353763415, "delta_retention_norm_ratio": 0.8950144399336618, "delta_cosine": 0.7147739387327632, "relative_weight_rmse": 0.011557893189828878 }, { "index": 68, "total": 252, "projection": "layers.17.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1355, "parameters": 16777216, "reference_delta_l2": 1.7297913669458336, "retained_delta_l2": 1.5698438793182137, "delta_retention_norm_ratio": 0.9075336536625064, "delta_cosine": 0.7247548434650827, "relative_weight_rmse": 0.01219956858762891 }, { "index": 69, "total": 252, "projection": "layers.17.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1356, "parameters": 16777216, "reference_delta_l2": 1.6613043369596119, "retained_delta_l2": 1.486608854445748, "delta_retention_norm_ratio": 0.8948443830384638, "delta_cosine": 0.7077833061431172, "relative_weight_rmse": 0.011971929027731735 }, { "index": 70, "total": 252, "projection": "layers.17.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0305, "parameters": 4194304, "reference_delta_l2": 1.033477168627319, "retained_delta_l2": 0.9691635128398266, "delta_retention_norm_ratio": 0.9377696404528078, "delta_cosine": 0.7589014017524195, "relative_weight_rmse": 0.01276351600474143 }, { "index": 71, "total": 252, "projection": "layers.18.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4376, "parameters": 50331648, "reference_delta_l2": 2.912291330744063, "retained_delta_l2": 2.624113222358469, "delta_retention_norm_ratio": 0.9010476371840288, "delta_cosine": 0.7155707881308109, "relative_weight_rmse": 0.011719327332632453 }, { "index": 72, "total": 252, "projection": "layers.18.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3934, "parameters": 50331648, "reference_delta_l2": 4.7323895908701195, "retained_delta_l2": 4.694887208565627, "delta_retention_norm_ratio": 0.9920753814570036, "delta_cosine": 0.8274217190394803, "relative_weight_rmse": 0.01543821079242807 }, { "index": 73, "total": 252, "projection": "layers.18.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.395, "parameters": 50331648, "reference_delta_l2": 3.2684049511531974, "retained_delta_l2": 3.007384330921114, "delta_retention_norm_ratio": 0.920138225179231, "delta_cosine": 0.7367400566488568, "relative_weight_rmse": 0.01248984104763899 }, { "index": 74, "total": 252, "projection": "layers.18.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.1176756575337732, "retained_delta_l2": 1.0843334020518682, "delta_retention_norm_ratio": 0.9701682189666035, "delta_cosine": 0.7990453769658297, "relative_weight_rmse": 0.013997563751336657 }, { "index": 75, "total": 252, "projection": "layers.18.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1349, "parameters": 16777216, "reference_delta_l2": 1.6285698839730298, "retained_delta_l2": 1.4555649563408954, "delta_retention_norm_ratio": 0.8937688033318689, "delta_cosine": 0.7107610372529769, "relative_weight_rmse": 0.01169664296587917 }, { "index": 76, "total": 252, "projection": "layers.18.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1344, "parameters": 16777216, "reference_delta_l2": 2.099547953488129, "retained_delta_l2": 2.009462095351522, "delta_retention_norm_ratio": 0.9570927360878131, "delta_cosine": 0.7836973547458617, "relative_weight_rmse": 0.013543508405450629 }, { "index": 77, "total": 252, "projection": "layers.18.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 0.9280061897807907, "retained_delta_l2": 0.8453754017599368, "delta_retention_norm_ratio": 0.9109587964705573, "delta_cosine": 0.7279125852473827, "relative_weight_rmse": 0.011785830425009313 }, { "index": 78, "total": 252, "projection": "layers.19.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4367, "parameters": 50331648, "reference_delta_l2": 2.6905658774983516, "retained_delta_l2": 2.353839107682188, "delta_retention_norm_ratio": 0.8748490893189922, "delta_cosine": 0.6890581511290075, "relative_weight_rmse": 0.011200443943313133 }, { "index": 79, "total": 252, "projection": "layers.19.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3929, "parameters": 50331648, "reference_delta_l2": 4.618692339064488, "retained_delta_l2": 4.552187654151666, "delta_retention_norm_ratio": 0.9856009710042102, "delta_cosine": 0.8186987969035592, "relative_weight_rmse": 0.01545184839911245 }, { "index": 80, "total": 252, "projection": "layers.19.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3936, "parameters": 50331648, "reference_delta_l2": 3.091580020506551, "retained_delta_l2": 2.7719047410257023, "delta_retention_norm_ratio": 0.8965980898568265, "delta_cosine": 0.7108192278959599, "relative_weight_rmse": 0.012195071398963545 }, { "index": 81, "total": 252, "projection": "layers.19.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 1.0441969278779464, "retained_delta_l2": 1.0049150954003139, "delta_retention_norm_ratio": 0.9623808197200289, "delta_cosine": 0.7951310937727945, "relative_weight_rmse": 0.013587193054594958 }, { "index": 82, "total": 252, "projection": "layers.19.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1349, "parameters": 16777216, "reference_delta_l2": 1.6398227134396683, "retained_delta_l2": 1.4603600255517262, "delta_retention_norm_ratio": 0.890559701108479, "delta_cosine": 0.705257501501427, "relative_weight_rmse": 0.011793715018796351 }, { "index": 83, "total": 252, "projection": "layers.19.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1347, "parameters": 16777216, "reference_delta_l2": 1.933861545575777, "retained_delta_l2": 1.8238676760994077, "delta_retention_norm_ratio": 0.9431221590149462, "delta_cosine": 0.7653721358101522, "relative_weight_rmse": 0.012813601727524927 }, { "index": 84, "total": 252, "projection": "layers.19.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0303, "parameters": 4194304, "reference_delta_l2": 1.2392430066984266, "retained_delta_l2": 1.2062007950941982, "delta_retention_norm_ratio": 0.9733367778348342, "delta_cosine": 0.806179559805123, "relative_weight_rmse": 0.013897893803669462 }, { "index": 85, "total": 252, "projection": "layers.2.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.438, "parameters": 50331648, "reference_delta_l2": 1.5500714165419396, "retained_delta_l2": 1.3349876126090936, "delta_retention_norm_ratio": 0.8612426487982874, "delta_cosine": 0.697999976458453, "relative_weight_rmse": 0.01010100203213799 }, { "index": 86, "total": 252, "projection": "layers.2.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.4065, "parameters": 50331648, "reference_delta_l2": 2.504646258958508, "retained_delta_l2": 2.3609978273546726, "delta_retention_norm_ratio": 0.9426472177098703, "delta_cosine": 0.7970291686137979, "relative_weight_rmse": 0.012648342970186247 }, { "index": 87, "total": 252, "projection": "layers.2.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3935, "parameters": 50331648, "reference_delta_l2": 3.3696103191055706, "retained_delta_l2": 3.349766906459199, "delta_retention_norm_ratio": 0.9941110660381528, "delta_cosine": 0.8793504353476361, "relative_weight_rmse": 0.014958809364992352 }, { "index": 88, "total": 252, "projection": "layers.2.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0315, "parameters": 4194304, "reference_delta_l2": 0.5769582349477705, "retained_delta_l2": 0.43406403930005427, "delta_retention_norm_ratio": 0.7523318205855024, "delta_cosine": 0.5760685043835092, "relative_weight_rmse": 0.00928746344440853 }, { "index": 89, "total": 252, "projection": "layers.2.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1349, "parameters": 16777216, "reference_delta_l2": 2.083701722857339, "retained_delta_l2": 1.9650056104568143, "delta_retention_norm_ratio": 0.9430359388301705, "delta_cosine": 0.7602686010254693, "relative_weight_rmse": 0.013939342093591264 }, { "index": 90, "total": 252, "projection": "layers.2.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1336, "parameters": 16777216, "reference_delta_l2": 1.2288907212918998, "retained_delta_l2": 0.9707183587780293, "delta_retention_norm_ratio": 0.7899143039810238, "delta_cosine": 0.608056427978517, "relative_weight_rmse": 0.010180997687138508 }, { "index": 91, "total": 252, "projection": "layers.2.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 1.1606500649521356, "retained_delta_l2": 1.119722100274245, "delta_retention_norm_ratio": 0.9647370332249295, "delta_cosine": 0.7932427944826372, "relative_weight_rmse": 0.013508958069442129 }, { "index": 92, "total": 252, "projection": "layers.20.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4371, "parameters": 50331648, "reference_delta_l2": 2.647556093950071, "retained_delta_l2": 2.2599685058612993, "delta_retention_norm_ratio": 0.8536055236093212, "delta_cosine": 0.6660049619390129, "relative_weight_rmse": 0.011252847750215126 }, { "index": 93, "total": 252, "projection": "layers.20.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3929, "parameters": 50331648, "reference_delta_l2": 4.54164391219833, "retained_delta_l2": 4.472502244452568, "delta_retention_norm_ratio": 0.9847760702770961, "delta_cosine": 0.8167110275761873, "relative_weight_rmse": 0.015413764053976025 }, { "index": 94, "total": 252, "projection": "layers.20.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3911, "parameters": 50331648, "reference_delta_l2": 3.1240951518173383, "retained_delta_l2": 2.8064881224754443, "delta_retention_norm_ratio": 0.898336313746034, "delta_cosine": 0.7121239647325682, "relative_weight_rmse": 0.01227175571033149 }, { "index": 95, "total": 252, "projection": "layers.20.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 0.8435434130294448, "retained_delta_l2": 0.7755878258928495, "delta_retention_norm_ratio": 0.9194403203356847, "delta_cosine": 0.7399829493130454, "relative_weight_rmse": 0.01219344130971694 }, { "index": 96, "total": 252, "projection": "layers.20.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1345, "parameters": 16777216, "reference_delta_l2": 1.60998213071129, "retained_delta_l2": 1.4290075961452402, "delta_retention_norm_ratio": 0.887592209184275, "delta_cosine": 0.703519942844837, "relative_weight_rmse": 0.011826373126148216 }, { "index": 97, "total": 252, "projection": "layers.20.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1354, "parameters": 16777216, "reference_delta_l2": 1.893648789853392, "retained_delta_l2": 1.7698240053263536, "delta_retention_norm_ratio": 0.9346104804700216, "delta_cosine": 0.757096633151059, "relative_weight_rmse": 0.012342558871086262 }, { "index": 98, "total": 252, "projection": "layers.20.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 0.9448415236568111, "retained_delta_l2": 0.8670623666086616, "delta_retention_norm_ratio": 0.9176802086903193, "delta_cosine": 0.7371462243551566, "relative_weight_rmse": 0.012026095186860188 }, { "index": 99, "total": 252, "projection": "layers.21.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4349, "parameters": 50331648, "reference_delta_l2": 3.105276205896329, "retained_delta_l2": 2.8032148298004578, "delta_retention_norm_ratio": 0.9027264062622469, "delta_cosine": 0.7157920387866503, "relative_weight_rmse": 0.012305052366142202 }, { "index": 100, "total": 252, "projection": "layers.21.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3908, "parameters": 50331648, "reference_delta_l2": 4.619770502917215, "retained_delta_l2": 4.556478246695928, "delta_retention_norm_ratio": 0.986299696882925, "delta_cosine": 0.8167262470911177, "relative_weight_rmse": 0.01590804096656067 }, { "index": 101, "total": 252, "projection": "layers.21.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3915, "parameters": 50331648, "reference_delta_l2": 3.3382291691770045, "retained_delta_l2": 3.0642641788203435, "delta_retention_norm_ratio": 0.9179310417372564, "delta_cosine": 0.7328829388695662, "relative_weight_rmse": 0.012751954415997446 }, { "index": 102, "total": 252, "projection": "layers.21.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.031, "parameters": 4194304, "reference_delta_l2": 0.889886028991794, "retained_delta_l2": 0.8231915337286541, "delta_retention_norm_ratio": 0.9250527673316749, "delta_cosine": 0.7423830456244178, "relative_weight_rmse": 0.012826996818041645 }, { "index": 103, "total": 252, "projection": "layers.21.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1337, "parameters": 16777216, "reference_delta_l2": 1.7388208776579452, "retained_delta_l2": 1.590756945940496, "delta_retention_norm_ratio": 0.9148480826174115, "delta_cosine": 0.7325991137758434, "relative_weight_rmse": 0.012392239183371758 }, { "index": 104, "total": 252, "projection": "layers.21.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1335, "parameters": 16777216, "reference_delta_l2": 1.346426603628278, "retained_delta_l2": 1.1134869039593562, "delta_retention_norm_ratio": 0.8269941346663766, "delta_cosine": 0.6436933610520533, "relative_weight_rmse": 0.010330564164912478 }, { "index": 105, "total": 252, "projection": "layers.21.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0309, "parameters": 4194304, "reference_delta_l2": 1.341526986386814, "retained_delta_l2": 1.3248957647642399, "delta_retention_norm_ratio": 0.9876027677479917, "delta_cosine": 0.8234711888276329, "relative_weight_rmse": 0.014563529675619849 }, { "index": 106, "total": 252, "projection": "layers.22.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4313, "parameters": 50331648, "reference_delta_l2": 3.360717108118368, "retained_delta_l2": 3.125093900153933, "delta_retention_norm_ratio": 0.9298890086894703, "delta_cosine": 0.7474962910334413, "relative_weight_rmse": 0.01260631904480232 }, { "index": 107, "total": 252, "projection": "layers.22.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.389, "parameters": 50331648, "reference_delta_l2": 4.210747269890564, "retained_delta_l2": 4.096031547804069, "delta_retention_norm_ratio": 0.9727564456535345, "delta_cosine": 0.8019330786138767, "relative_weight_rmse": 0.014651000649681535 }, { "index": 108, "total": 252, "projection": "layers.22.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3913, "parameters": 50331648, "reference_delta_l2": 3.4252916600455015, "retained_delta_l2": 3.174494875191197, "delta_retention_norm_ratio": 0.9267808964183293, "delta_cosine": 0.7449150154482405, "relative_weight_rmse": 0.012618399403942023 }, { "index": 109, "total": 252, "projection": "layers.22.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.1256697925354247, "retained_delta_l2": 1.1006366382431394, "delta_retention_norm_ratio": 0.9777615474286636, "delta_cosine": 0.817683777268333, "relative_weight_rmse": 0.014420613901313934 }, { "index": 110, "total": 252, "projection": "layers.22.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1345, "parameters": 16777216, "reference_delta_l2": 2.099753726538612, "retained_delta_l2": 1.999529306975156, "delta_retention_norm_ratio": 0.9522684883009239, "delta_cosine": 0.7730058820038571, "relative_weight_rmse": 0.01373655270061581 }, { "index": 111, "total": 252, "projection": "layers.22.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1346, "parameters": 16777216, "reference_delta_l2": 1.8696581267064718, "retained_delta_l2": 1.740177726303158, "delta_retention_norm_ratio": 0.9307464832453608, "delta_cosine": 0.7545627126669409, "relative_weight_rmse": 0.01259379142506784 }, { "index": 112, "total": 252, "projection": "layers.22.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 1.10357555374315, "retained_delta_l2": 1.0512816270257077, "delta_retention_norm_ratio": 0.9526140946670395, "delta_cosine": 0.7775259204534436, "relative_weight_rmse": 0.013222044058572085 }, { "index": 113, "total": 252, "projection": "layers.23.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4311, "parameters": 50331648, "reference_delta_l2": 3.160872318390849, "retained_delta_l2": 2.9013703143071194, "delta_retention_norm_ratio": 0.9179017758566602, "delta_cosine": 0.73519129883951, "relative_weight_rmse": 0.011974311913947348 }, { "index": 114, "total": 252, "projection": "layers.23.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.39, "parameters": 50331648, "reference_delta_l2": 5.633661482659359, "retained_delta_l2": 5.677116194607454, "delta_retention_norm_ratio": 1.0077134048756482, "delta_cosine": 0.8533048383623517, "relative_weight_rmse": 0.016809090155428346 }, { "index": 115, "total": 252, "projection": "layers.23.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3924, "parameters": 50331648, "reference_delta_l2": 4.980404555262815, "retained_delta_l2": 4.942861482350325, "delta_retention_norm_ratio": 0.992461842708577, "delta_cosine": 0.828900774170685, "relative_weight_rmse": 0.015409840480461122 }, { "index": 116, "total": 252, "projection": "layers.23.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0316, "parameters": 4194304, "reference_delta_l2": 0.7947115673087248, "retained_delta_l2": 0.7073865528627679, "delta_retention_norm_ratio": 0.8901173481824589, "delta_cosine": 0.7057249262457971, "relative_weight_rmse": 0.011975734018331215 }, { "index": 117, "total": 252, "projection": "layers.23.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1365, "parameters": 16777216, "reference_delta_l2": 1.961022080361704, "retained_delta_l2": 1.8342382011607155, "delta_retention_norm_ratio": 0.9353480613652226, "delta_cosine": 0.7548158970571475, "relative_weight_rmse": 0.012808345048691697 }, { "index": 118, "total": 252, "projection": "layers.23.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1343, "parameters": 16777216, "reference_delta_l2": 1.7869979322140728, "retained_delta_l2": 1.6195382621873493, "delta_retention_norm_ratio": 0.906289947510324, "delta_cosine": 0.7240350295402113, "relative_weight_rmse": 0.012030264827546133 }, { "index": 119, "total": 252, "projection": "layers.23.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0301, "parameters": 4194304, "reference_delta_l2": 0.9269590107602388, "retained_delta_l2": 0.8512280843157533, "delta_retention_norm_ratio": 0.9183017527577888, "delta_cosine": 0.7376277684845678, "relative_weight_rmse": 0.011725215372115768 }, { "index": 120, "total": 252, "projection": "layers.24.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4305, "parameters": 50331648, "reference_delta_l2": 3.2503102452109283, "retained_delta_l2": 2.978091014440522, "delta_retention_norm_ratio": 0.9162482316352727, "delta_cosine": 0.7315355094947241, "relative_weight_rmse": 0.01227254580627398 }, { "index": 121, "total": 252, "projection": "layers.24.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3947, "parameters": 50331648, "reference_delta_l2": 5.122346784643699, "retained_delta_l2": 5.090735605570083, "delta_retention_norm_ratio": 0.9938287702097051, "delta_cosine": 0.8272340568179678, "relative_weight_rmse": 0.01624333933890823 }, { "index": 122, "total": 252, "projection": "layers.24.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3933, "parameters": 50331648, "reference_delta_l2": 4.007395782761482, "retained_delta_l2": 3.8291209124834666, "delta_retention_norm_ratio": 0.9555135354873368, "delta_cosine": 0.7755254983957733, "relative_weight_rmse": 0.013944425880539327 }, { "index": 123, "total": 252, "projection": "layers.24.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.9660765277735609, "retained_delta_l2": 0.9167020992533299, "delta_retention_norm_ratio": 0.9488918040126486, "delta_cosine": 0.7724223783247411, "relative_weight_rmse": 0.013150187824250323 }, { "index": 124, "total": 252, "projection": "layers.24.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.135, "parameters": 16777216, "reference_delta_l2": 1.918328876575902, "retained_delta_l2": 1.779257488401996, "delta_retention_norm_ratio": 0.9275038863919206, "delta_cosine": 0.7461881577671706, "relative_weight_rmse": 0.012580540298291065 }, { "index": 125, "total": 252, "projection": "layers.24.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1348, "parameters": 16777216, "reference_delta_l2": 3.0209953608095272, "retained_delta_l2": 3.0175153247508884, "delta_retention_norm_ratio": 0.9988480498501309, "delta_cosine": 0.8487291450530009, "relative_weight_rmse": 0.015555315896263225 }, { "index": 126, "total": 252, "projection": "layers.24.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 1.0885553070106562, "retained_delta_l2": 1.0372228366876877, "delta_retention_norm_ratio": 0.9528434889873116, "delta_cosine": 0.7791866473600985, "relative_weight_rmse": 0.013054892107370254 }, { "index": 127, "total": 252, "projection": "layers.25.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4331, "parameters": 50331648, "reference_delta_l2": 3.6373147216160677, "retained_delta_l2": 3.404055364663721, "delta_retention_norm_ratio": 0.9358704498221949, "delta_cosine": 0.7514459847745487, "relative_weight_rmse": 0.013263712821630426 }, { "index": 128, "total": 252, "projection": "layers.25.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3927, "parameters": 50331648, "reference_delta_l2": 5.600910492578175, "retained_delta_l2": 5.614913656800555, "delta_retention_norm_ratio": 1.0025001585440323, "delta_cosine": 0.8406864774637491, "relative_weight_rmse": 0.016952561312630426 }, { "index": 129, "total": 252, "projection": "layers.25.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3946, "parameters": 50331648, "reference_delta_l2": 3.745221022962041, "retained_delta_l2": 3.49549093622834, "delta_retention_norm_ratio": 0.9333203340463487, "delta_cosine": 0.7482711929833467, "relative_weight_rmse": 0.013621479587259277 }, { "index": 130, "total": 252, "projection": "layers.25.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 1.0289437223960307, "retained_delta_l2": 0.982520413562484, "delta_retention_norm_ratio": 0.9548825578862138, "delta_cosine": 0.7800469586103691, "relative_weight_rmse": 0.013545644703222918 }, { "index": 131, "total": 252, "projection": "layers.25.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1346, "parameters": 16777216, "reference_delta_l2": 1.9553584720962225, "retained_delta_l2": 1.8082254096930057, "delta_retention_norm_ratio": 0.9247539187812022, "delta_cosine": 0.7401622947458671, "relative_weight_rmse": 0.013195673526209827 }, { "index": 132, "total": 252, "projection": "layers.25.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1314, "parameters": 16777216, "reference_delta_l2": 2.450296786092303, "retained_delta_l2": 2.402918629425209, "delta_retention_norm_ratio": 0.980664319140437, "delta_cosine": 0.8152132178531424, "relative_weight_rmse": 0.014595703215884949 }, { "index": 133, "total": 252, "projection": "layers.25.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0302, "parameters": 4194304, "reference_delta_l2": 1.1880491684330374, "retained_delta_l2": 1.1446621680107196, "delta_retention_norm_ratio": 0.9634804673281809, "delta_cosine": 0.7872418634950002, "relative_weight_rmse": 0.01345733935618882 }, { "index": 134, "total": 252, "projection": "layers.26.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4608, "parameters": 50331648, "reference_delta_l2": 3.2957611727198537, "retained_delta_l2": 3.016838300593981, "delta_retention_norm_ratio": 0.9153692098703593, "delta_cosine": 0.7283018254489741, "relative_weight_rmse": 0.012337697827002952 }, { "index": 135, "total": 252, "projection": "layers.26.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.4003, "parameters": 50331648, "reference_delta_l2": 5.260130561195044, "retained_delta_l2": 5.222515082979985, "delta_retention_norm_ratio": 0.9928489459002109, "delta_cosine": 0.8221569371622288, "relative_weight_rmse": 0.016656073965757832 }, { "index": 136, "total": 252, "projection": "layers.26.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.4084, "parameters": 50331648, "reference_delta_l2": 3.598925250679574, "retained_delta_l2": 3.3267956511150807, "delta_retention_norm_ratio": 0.9243858706114811, "delta_cosine": 0.7381639234093447, "relative_weight_rmse": 0.013167427550091431 }, { "index": 137, "total": 252, "projection": "layers.26.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0321, "parameters": 4194304, "reference_delta_l2": 1.102208846410291, "retained_delta_l2": 1.0639393919165272, "delta_retention_norm_ratio": 0.9652793074394194, "delta_cosine": 0.7913675307064206, "relative_weight_rmse": 0.014438720269170098 }, { "index": 138, "total": 252, "projection": "layers.26.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1349, "parameters": 16777216, "reference_delta_l2": 2.145429788307753, "retained_delta_l2": 2.021116465396612, "delta_retention_norm_ratio": 0.9420566808624414, "delta_cosine": 0.7604249328209329, "relative_weight_rmse": 0.013810083511670299 }, { "index": 139, "total": 252, "projection": "layers.26.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1332, "parameters": 16777216, "reference_delta_l2": 2.7116328068071667, "retained_delta_l2": 2.6872061495663733, "delta_retention_norm_ratio": 0.9909919008283593, "delta_cosine": 0.8318779690607394, "relative_weight_rmse": 0.015357608678216816 }, { "index": 140, "total": 252, "projection": "layers.26.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0303, "parameters": 4194304, "reference_delta_l2": 1.300912100078543, "retained_delta_l2": 1.2655509068920476, "delta_retention_norm_ratio": 0.9728181533676561, "delta_cosine": 0.8069024380143299, "relative_weight_rmse": 0.014256978492757717 }, { "index": 141, "total": 252, "projection": "layers.27.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4441, "parameters": 50331648, "reference_delta_l2": 3.1221671631429038, "retained_delta_l2": 2.7953025124195756, "delta_retention_norm_ratio": 0.8953084080244145, "delta_cosine": 0.7077534475689679, "relative_weight_rmse": 0.011909876215115344 }, { "index": 142, "total": 252, "projection": "layers.27.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3914, "parameters": 50331648, "reference_delta_l2": 5.226479588987116, "retained_delta_l2": 5.1933838720565655, "delta_retention_norm_ratio": 0.9936676846494746, "delta_cosine": 0.8248796468505778, "relative_weight_rmse": 0.01645056821096251 }, { "index": 143, "total": 252, "projection": "layers.27.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3904, "parameters": 50331648, "reference_delta_l2": 3.1439875550600127, "retained_delta_l2": 2.7609244012810183, "delta_retention_norm_ratio": 0.8781600922171328, "delta_cosine": 0.6898881540050043, "relative_weight_rmse": 0.012184674639886598 }, { "index": 144, "total": 252, "projection": "layers.27.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.032, "parameters": 4194304, "reference_delta_l2": 0.9578251695993613, "retained_delta_l2": 0.9016073643405902, "delta_retention_norm_ratio": 0.9413068198215278, "delta_cosine": 0.7665743632754864, "relative_weight_rmse": 0.013498560985661111 }, { "index": 145, "total": 252, "projection": "layers.27.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1343, "parameters": 16777216, "reference_delta_l2": 1.834174655123338, "retained_delta_l2": 1.6691012027622767, "delta_retention_norm_ratio": 0.9100012357602002, "delta_cosine": 0.7239421816255456, "relative_weight_rmse": 0.012743272136643499 }, { "index": 146, "total": 252, "projection": "layers.27.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1336, "parameters": 16777216, "reference_delta_l2": 2.5051869458196565, "retained_delta_l2": 2.450649274322715, "delta_retention_norm_ratio": 0.9782300991197695, "delta_cosine": 0.8133656660897193, "relative_weight_rmse": 0.014896108715641143 }, { "index": 147, "total": 252, "projection": "layers.27.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0304, "parameters": 4194304, "reference_delta_l2": 1.305157478619902, "retained_delta_l2": 1.2820106051062443, "delta_retention_norm_ratio": 0.9822650723051953, "delta_cosine": 0.814217592620622, "relative_weight_rmse": 0.014311472366894704 }, { "index": 148, "total": 252, "projection": "layers.28.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4307, "parameters": 50331648, "reference_delta_l2": 3.7617323107123806, "retained_delta_l2": 3.5468414125548615, "delta_retention_norm_ratio": 0.9428744843045932, "delta_cosine": 0.7610179061584897, "relative_weight_rmse": 0.013099953206764646 }, { "index": 149, "total": 252, "projection": "layers.28.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3892, "parameters": 50331648, "reference_delta_l2": 6.423387439542335, "retained_delta_l2": 6.537355606301714, "delta_retention_norm_ratio": 1.0177426891701709, "delta_cosine": 0.8668797945436078, "relative_weight_rmse": 0.017944310239398756 }, { "index": 150, "total": 252, "projection": "layers.28.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3942, "parameters": 50331648, "reference_delta_l2": 3.243137758540773, "retained_delta_l2": 2.875504229529378, "delta_retention_norm_ratio": 0.8866426416690949, "delta_cosine": 0.6978610398465285, "relative_weight_rmse": 0.01232328346889927 }, { "index": 151, "total": 252, "projection": "layers.28.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 1.0316368625723313, "retained_delta_l2": 0.9816314502213522, "delta_retention_norm_ratio": 0.9515280869023105, "delta_cosine": 0.7709917053597658, "relative_weight_rmse": 0.014130999330449982 }, { "index": 152, "total": 252, "projection": "layers.28.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1354, "parameters": 16777216, "reference_delta_l2": 1.8674884937261969, "retained_delta_l2": 1.715844398314334, "delta_retention_norm_ratio": 0.9187978421707502, "delta_cosine": 0.7338063460605813, "relative_weight_rmse": 0.012833414630404159 }, { "index": 153, "total": 252, "projection": "layers.28.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1345, "parameters": 16777216, "reference_delta_l2": 2.1062141931540292, "retained_delta_l2": 2.0063063710901976, "delta_retention_norm_ratio": 0.9525652127933762, "delta_cosine": 0.7769483273433703, "relative_weight_rmse": 0.013246534383223457 }, { "index": 154, "total": 252, "projection": "layers.28.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 1.0568340113396937, "retained_delta_l2": 0.9874382825117356, "delta_retention_norm_ratio": 0.9343362078781051, "delta_cosine": 0.7518618664441322, "relative_weight_rmse": 0.012582377681010531 }, { "index": 155, "total": 252, "projection": "layers.29.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.6148, "parameters": 50331648, "reference_delta_l2": 3.7401416658072963, "retained_delta_l2": 3.5095521416446185, "delta_retention_norm_ratio": 0.9383473823275873, "delta_cosine": 0.7558945279178694, "relative_weight_rmse": 0.012997864369369908 }, { "index": 156, "total": 252, "projection": "layers.29.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.516, "parameters": 50331648, "reference_delta_l2": 6.127935986595271, "retained_delta_l2": 6.212127194514415, "delta_retention_norm_ratio": 1.0137389176556855, "delta_cosine": 0.8568512009900341, "relative_weight_rmse": 0.017714506498618523 }, { "index": 157, "total": 252, "projection": "layers.29.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3926, "parameters": 50331648, "reference_delta_l2": 3.580009790840115, "retained_delta_l2": 3.2798771313174777, "delta_retention_norm_ratio": 0.9161642908657504, "delta_cosine": 0.7293571040606517, "relative_weight_rmse": 0.01289677374486609 }, { "index": 158, "total": 252, "projection": "layers.29.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0309, "parameters": 4194304, "reference_delta_l2": 0.9620634543526848, "retained_delta_l2": 0.9117803875080858, "delta_retention_norm_ratio": 0.9477341472466266, "delta_cosine": 0.7817947687473311, "relative_weight_rmse": 0.013131358653178712 }, { "index": 159, "total": 252, "projection": "layers.29.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1346, "parameters": 16777216, "reference_delta_l2": 1.9458139633919214, "retained_delta_l2": 1.79409683508142, "delta_retention_norm_ratio": 0.9220289651709407, "delta_cosine": 0.7376047556088177, "relative_weight_rmse": 0.01293095805509277 }, { "index": 160, "total": 252, "projection": "layers.29.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.134, "parameters": 16777216, "reference_delta_l2": 2.9427336589969983, "retained_delta_l2": 2.929736370121851, "delta_retention_norm_ratio": 0.9955832601990976, "delta_cosine": 0.8477660174783292, "relative_weight_rmse": 0.015273119054114149 }, { "index": 161, "total": 252, "projection": "layers.29.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 1.0690905177656662, "retained_delta_l2": 1.0095726237992413, "delta_retention_norm_ratio": 0.9443284801638558, "delta_cosine": 0.7674733040578655, "relative_weight_rmse": 0.012951905130350124 }, { "index": 162, "total": 252, "projection": "layers.3.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4443, "parameters": 50331648, "reference_delta_l2": 2.0679400082790367, "retained_delta_l2": 1.8308734297139297, "delta_retention_norm_ratio": 0.885360998086982, "delta_cosine": 0.7163025161839787, "relative_weight_rmse": 0.011341012968361944 }, { "index": 163, "total": 252, "projection": "layers.3.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3974, "parameters": 50331648, "reference_delta_l2": 2.9244244667822037, "retained_delta_l2": 2.702986678376405, "delta_retention_norm_ratio": 0.9242798742381435, "delta_cosine": 0.7458795583880982, "relative_weight_rmse": 0.012253597562551328 }, { "index": 164, "total": 252, "projection": "layers.3.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3976, "parameters": 50331648, "reference_delta_l2": 3.1770019858721863, "retained_delta_l2": 3.1111471145462324, "delta_retention_norm_ratio": 0.979271378608259, "delta_cosine": 0.8422329906912724, "relative_weight_rmse": 0.014673401123906079 }, { "index": 165, "total": 252, "projection": "layers.3.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0331, "parameters": 4194304, "reference_delta_l2": 0.5944881406399625, "retained_delta_l2": 0.4613283113924287, "delta_retention_norm_ratio": 0.7760092756363681, "delta_cosine": 0.5969925389093143, "relative_weight_rmse": 0.009636678973232336 }, { "index": 166, "total": 252, "projection": "layers.3.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.138, "parameters": 16777216, "reference_delta_l2": 1.8542744034159462, "retained_delta_l2": 1.7012318629959648, "delta_retention_norm_ratio": 0.9174649986334027, "delta_cosine": 0.7309382652239491, "relative_weight_rmse": 0.01283209216131865 }, { "index": 167, "total": 252, "projection": "layers.3.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1356, "parameters": 16777216, "reference_delta_l2": 1.509164806161759, "retained_delta_l2": 1.3064463511188427, "delta_retention_norm_ratio": 0.8656750712611118, "delta_cosine": 0.6791005299483913, "relative_weight_rmse": 0.01172984899046623 }, { "index": 168, "total": 252, "projection": "layers.3.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0307, "parameters": 4194304, "reference_delta_l2": 0.9490041062383832, "retained_delta_l2": 0.8558240584115787, "delta_retention_norm_ratio": 0.9018128085913694, "delta_cosine": 0.7160810428968206, "relative_weight_rmse": 0.012407708147450661 }, { "index": 169, "total": 252, "projection": "layers.30.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4365, "parameters": 50331648, "reference_delta_l2": 3.887870797219381, "retained_delta_l2": 3.6874548470093127, "delta_retention_norm_ratio": 0.9484509746688581, "delta_cosine": 0.7666196798290923, "relative_weight_rmse": 0.013139021782059881 }, { "index": 170, "total": 252, "projection": "layers.30.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3966, "parameters": 50331648, "reference_delta_l2": 5.399243984840606, "retained_delta_l2": 5.396158107286254, "delta_retention_norm_ratio": 0.9994284611765988, "delta_cosine": 0.8307007253136764, "relative_weight_rmse": 0.016931017999366662 }, { "index": 171, "total": 252, "projection": "layers.30.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3967, "parameters": 50331648, "reference_delta_l2": 3.809190933435205, "retained_delta_l2": 3.5287988984779015, "delta_retention_norm_ratio": 0.9263906588414458, "delta_cosine": 0.7397053682401827, "relative_weight_rmse": 0.013403786510319985 }, { "index": 172, "total": 252, "projection": "layers.30.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.495041637306258, "retained_delta_l2": 1.5068418452467638, "delta_retention_norm_ratio": 1.0078928958539024, "delta_cosine": 0.8527559337476618, "relative_weight_rmse": 0.016604996721110866 }, { "index": 173, "total": 252, "projection": "layers.30.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1353, "parameters": 16777216, "reference_delta_l2": 1.8684826155757526, "retained_delta_l2": 1.6923697735085421, "delta_retention_norm_ratio": 0.9057455281632667, "delta_cosine": 0.717530868631098, "relative_weight_rmse": 0.012862563146356202 }, { "index": 174, "total": 252, "projection": "layers.30.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1337, "parameters": 16777216, "reference_delta_l2": 3.018528780814635, "retained_delta_l2": 3.0295039027640356, "delta_retention_norm_ratio": 1.0036359176096503, "delta_cosine": 0.8550451128261393, "relative_weight_rmse": 0.016293838818625218 }, { "index": 175, "total": 252, "projection": "layers.30.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 1.3092329397348734, "retained_delta_l2": 1.2710158221445458, "delta_retention_norm_ratio": 0.9708095355452431, "delta_cosine": 0.8014254919574851, "relative_weight_rmse": 0.013918002500388403 }, { "index": 176, "total": 252, "projection": "layers.31.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4388, "parameters": 50331648, "reference_delta_l2": 4.110364027266316, "retained_delta_l2": 3.9447523499479558, "delta_retention_norm_ratio": 0.959708756640588, "delta_cosine": 0.7805102658039987, "relative_weight_rmse": 0.013378582867462649 }, { "index": 177, "total": 252, "projection": "layers.31.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.4003, "parameters": 50331648, "reference_delta_l2": 4.213688960532953, "retained_delta_l2": 4.065268359369619, "delta_retention_norm_ratio": 0.9647765645367994, "delta_cosine": 0.7857099884712581, "relative_weight_rmse": 0.014756743435253107 }, { "index": 178, "total": 252, "projection": "layers.31.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3951, "parameters": 50331648, "reference_delta_l2": 3.7452434781817034, "retained_delta_l2": 3.457631741618175, "delta_retention_norm_ratio": 0.92320613112631, "delta_cosine": 0.7366745284849822, "relative_weight_rmse": 0.013124970861274277 }, { "index": 179, "total": 252, "projection": "layers.31.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0322, "parameters": 4194304, "reference_delta_l2": 1.131255621313341, "retained_delta_l2": 1.1076962069633571, "delta_retention_norm_ratio": 0.9791741018509749, "delta_cosine": 0.8129665759244713, "relative_weight_rmse": 0.014434867123085856 }, { "index": 180, "total": 252, "projection": "layers.31.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1346, "parameters": 16777216, "reference_delta_l2": 1.9271778268048494, "retained_delta_l2": 1.7785369541374052, "delta_retention_norm_ratio": 0.9228712210155082, "delta_cosine": 0.738814993615326, "relative_weight_rmse": 0.012540620968805942 }, { "index": 181, "total": 252, "projection": "layers.31.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1351, "parameters": 16777216, "reference_delta_l2": 3.0951075436016566, "retained_delta_l2": 3.1186091997514307, "delta_retention_norm_ratio": 1.0075931630221888, "delta_cosine": 0.8593239195692756, "relative_weight_rmse": 0.016359630296747883 }, { "index": 182, "total": 252, "projection": "layers.31.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0311, "parameters": 4194304, "reference_delta_l2": 0.913180736908778, "retained_delta_l2": 0.8056474687920849, "delta_retention_norm_ratio": 0.8822431707432797, "delta_cosine": 0.6994518824150215, "relative_weight_rmse": 0.011245913328725037 }, { "index": 183, "total": 252, "projection": "layers.32.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4373, "parameters": 50331648, "reference_delta_l2": 3.053233543999877, "retained_delta_l2": 2.7010398933001922, "delta_retention_norm_ratio": 0.8846489645733767, "delta_cosine": 0.6983303428179822, "relative_weight_rmse": 0.011288760006122339 }, { "index": 184, "total": 252, "projection": "layers.32.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3952, "parameters": 50331648, "reference_delta_l2": 5.735749275322441, "retained_delta_l2": 5.787299683307666, "delta_retention_norm_ratio": 1.008987563003672, "delta_cosine": 0.8483716330269929, "relative_weight_rmse": 0.01748269997937928 }, { "index": 185, "total": 252, "projection": "layers.32.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.393, "parameters": 50331648, "reference_delta_l2": 4.1791007274280325, "retained_delta_l2": 3.9628442817535, "delta_retention_norm_ratio": 0.9482528754917988, "delta_cosine": 0.7672786556742622, "relative_weight_rmse": 0.013836685178664192 }, { "index": 186, "total": 252, "projection": "layers.32.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0313, "parameters": 4194304, "reference_delta_l2": 1.0751946475751073, "retained_delta_l2": 1.040603625981545, "delta_retention_norm_ratio": 0.9678281307746688, "delta_cosine": 0.7986439823197274, "relative_weight_rmse": 0.014082435322431848 }, { "index": 187, "total": 252, "projection": "layers.32.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1355, "parameters": 16777216, "reference_delta_l2": 1.9526543310434923, "retained_delta_l2": 1.7946643532432096, "delta_retention_norm_ratio": 0.9190896333833682, "delta_cosine": 0.7327058975081081, "relative_weight_rmse": 0.012906716155433212 }, { "index": 188, "total": 252, "projection": "layers.32.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.135, "parameters": 16777216, "reference_delta_l2": 3.5037919178289854, "retained_delta_l2": 3.5634283880741373, "delta_retention_norm_ratio": 1.0170205513465833, "delta_cosine": 0.8837426455755633, "relative_weight_rmse": 0.017016034747910174 }, { "index": 189, "total": 252, "projection": "layers.32.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0307, "parameters": 4194304, "reference_delta_l2": 1.1042248941807926, "retained_delta_l2": 1.0345746225272665, "delta_retention_norm_ratio": 0.9369238349718617, "delta_cosine": 0.7579939795578423, "relative_weight_rmse": 0.01259764580426598 }, { "index": 190, "total": 252, "projection": "layers.33.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4356, "parameters": 50331648, "reference_delta_l2": 4.677230833346037, "retained_delta_l2": 4.590480414565143, "delta_retention_norm_ratio": 0.9814526111984015, "delta_cosine": 0.8097453894324198, "relative_weight_rmse": 0.01440073902379026 }, { "index": 191, "total": 252, "projection": "layers.33.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3933, "parameters": 50331648, "reference_delta_l2": 4.002110728928395, "retained_delta_l2": 3.8174978532167745, "delta_retention_norm_ratio": 0.9538711224611587, "delta_cosine": 0.7708522234085368, "relative_weight_rmse": 0.014726103170449745 }, { "index": 192, "total": 252, "projection": "layers.33.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3936, "parameters": 50331648, "reference_delta_l2": 4.549819665593091, "retained_delta_l2": 4.4131762772195, "delta_retention_norm_ratio": 0.9699672957574728, "delta_cosine": 0.7954954853276326, "relative_weight_rmse": 0.014223793742498481 }, { "index": 193, "total": 252, "projection": "layers.33.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 1.1715315410735023, "retained_delta_l2": 1.1622101639761016, "delta_retention_norm_ratio": 0.9920434262581961, "delta_cosine": 0.8277438557448046, "relative_weight_rmse": 0.015548853767619908 }, { "index": 194, "total": 252, "projection": "layers.33.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1386, "parameters": 16777216, "reference_delta_l2": 1.9572263469894886, "retained_delta_l2": 1.809083490347337, "delta_retention_norm_ratio": 0.9243097984706685, "delta_cosine": 0.7413618217608011, "relative_weight_rmse": 0.012300056453069263 }, { "index": 195, "total": 252, "projection": "layers.33.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1413, "parameters": 16777216, "reference_delta_l2": 1.695420696327585, "retained_delta_l2": 1.5631995893456434, "delta_retention_norm_ratio": 0.9220128035075053, "delta_cosine": 0.7427885960279792, "relative_weight_rmse": 0.011923084861626338 }, { "index": 196, "total": 252, "projection": "layers.33.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0307, "parameters": 4194304, "reference_delta_l2": 1.2334249104591484, "retained_delta_l2": 1.1838953351223982, "delta_retention_norm_ratio": 0.9598438665242195, "delta_cosine": 0.7863020889219451, "relative_weight_rmse": 0.01272197288675906 }, { "index": 197, "total": 252, "projection": "layers.34.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4345, "parameters": 50331648, "reference_delta_l2": 3.2554531412284744, "retained_delta_l2": 3.0244708407371674, "delta_retention_norm_ratio": 0.9290475732652862, "delta_cosine": 0.7454650945192635, "relative_weight_rmse": 0.011840481594971156 }, { "index": 198, "total": 252, "projection": "layers.34.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3928, "parameters": 50331648, "reference_delta_l2": 3.5119099908536424, "retained_delta_l2": 3.307978801417105, "delta_retention_norm_ratio": 0.9419315443825006, "delta_cosine": 0.7569929254989856, "relative_weight_rmse": 0.013105608965876779 }, { "index": 199, "total": 252, "projection": "layers.34.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3944, "parameters": 50331648, "reference_delta_l2": 3.1197603574733135, "retained_delta_l2": 2.795634123462441, "delta_retention_norm_ratio": 0.8961054065468729, "delta_cosine": 0.709619591445098, "relative_weight_rmse": 0.011414617605965967 }, { "index": 200, "total": 252, "projection": "layers.34.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0314, "parameters": 4194304, "reference_delta_l2": 0.7884807301773037, "retained_delta_l2": 0.7270568299201107, "delta_retention_norm_ratio": 0.9220984129271228, "delta_cosine": 0.7525313403172603, "relative_weight_rmse": 0.011626929601153246 }, { "index": 201, "total": 252, "projection": "layers.34.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1354, "parameters": 16777216, "reference_delta_l2": 2.3657537786912455, "retained_delta_l2": 2.295153793011888, "delta_retention_norm_ratio": 0.970157509071627, "delta_cosine": 0.7929635793398593, "relative_weight_rmse": 0.014041838832518524 }, { "index": 202, "total": 252, "projection": "layers.34.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1308, "parameters": 16777216, "reference_delta_l2": 2.7912707575141327, "retained_delta_l2": 2.8046778609760277, "delta_retention_norm_ratio": 1.0048032257085069, "delta_cosine": 0.859762626141908, "relative_weight_rmse": 0.015280208881268444 }, { "index": 203, "total": 252, "projection": "layers.34.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0303, "parameters": 4194304, "reference_delta_l2": 0.98146189599056, "retained_delta_l2": 0.8769177949999751, "delta_retention_norm_ratio": 0.8934812432172197, "delta_cosine": 0.7154066928079399, "relative_weight_rmse": 0.011405952287331642 }, { "index": 204, "total": 252, "projection": "layers.35.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4307, "parameters": 50331648, "reference_delta_l2": 2.323442105920221, "retained_delta_l2": 2.0756301760951703, "delta_retention_norm_ratio": 0.8933427567686683, "delta_cosine": 0.7136688211460611, "relative_weight_rmse": 0.009788394551676757 }, { "index": 205, "total": 252, "projection": "layers.35.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3882, "parameters": 50331648, "reference_delta_l2": 2.619391209776972, "retained_delta_l2": 2.284627070690156, "delta_retention_norm_ratio": 0.8721977313517366, "delta_cosine": 0.686894707285102, "relative_weight_rmse": 0.010512670834223626 }, { "index": 206, "total": 252, "projection": "layers.35.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3896, "parameters": 50331648, "reference_delta_l2": 2.658687967607658, "retained_delta_l2": 2.2856054411666, "delta_retention_norm_ratio": 0.8596741960747031, "delta_cosine": 0.6743087239399935, "relative_weight_rmse": 0.01033738841891372 }, { "index": 207, "total": 252, "projection": "layers.35.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 0.8329213911977184, "retained_delta_l2": 0.7629292702052147, "delta_retention_norm_ratio": 0.9159679151812192, "delta_cosine": 0.7342860229477369, "relative_weight_rmse": 0.01222969875625576 }, { "index": 208, "total": 252, "projection": "layers.35.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1319, "parameters": 16777216, "reference_delta_l2": 1.978736408751587, "retained_delta_l2": 1.877442477532796, "delta_retention_norm_ratio": 0.9488087797996809, "delta_cosine": 0.767618956994911, "relative_weight_rmse": 0.012986472755049936 }, { "index": 209, "total": 252, "projection": "layers.35.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1323, "parameters": 16777216, "reference_delta_l2": 1.9212411836563124, "retained_delta_l2": 1.8332294176962938, "delta_retention_norm_ratio": 0.9541901523303161, "delta_cosine": 0.7745190405033069, "relative_weight_rmse": 0.013081762644918027 }, { "index": 210, "total": 252, "projection": "layers.35.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0298, "parameters": 4194304, "reference_delta_l2": 0.9030694749944501, "retained_delta_l2": 0.7966562129589475, "delta_retention_norm_ratio": 0.8821649219888021, "delta_cosine": 0.6990966561194207, "relative_weight_rmse": 0.011406911676881168 }, { "index": 211, "total": 252, "projection": "layers.4.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4323, "parameters": 50331648, "reference_delta_l2": 2.2245974764067546, "retained_delta_l2": 1.9080686344231745, "delta_retention_norm_ratio": 0.8577141054323012, "delta_cosine": 0.6729869101107243, "relative_weight_rmse": 0.010835389092090777 }, { "index": 212, "total": 252, "projection": "layers.4.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3893, "parameters": 50331648, "reference_delta_l2": 3.314203046594274, "retained_delta_l2": 2.92327997457648, "delta_retention_norm_ratio": 0.882046130993841, "delta_cosine": 0.7090416043478848, "relative_weight_rmse": 0.010831238116930547 }, { "index": 213, "total": 252, "projection": "layers.4.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3878, "parameters": 50331648, "reference_delta_l2": 3.2992289651260265, "retained_delta_l2": 3.176838706079354, "delta_retention_norm_ratio": 0.9629033752005759, "delta_cosine": 0.7950672894455574, "relative_weight_rmse": 0.01401853766560735 }, { "index": 214, "total": 252, "projection": "layers.4.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0309, "parameters": 4194304, "reference_delta_l2": 0.5934329624541549, "retained_delta_l2": 0.4482821586618158, "delta_retention_norm_ratio": 0.7554048848381041, "delta_cosine": 0.5783625575635638, "relative_weight_rmse": 0.009466441979177278 }, { "index": 215, "total": 252, "projection": "layers.4.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1334, "parameters": 16777216, "reference_delta_l2": 1.6294614994627534, "retained_delta_l2": 1.4159643905718167, "delta_retention_norm_ratio": 0.8689768926965574, "delta_cosine": 0.6811809510270403, "relative_weight_rmse": 0.011808513059008886 }, { "index": 216, "total": 252, "projection": "layers.4.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1339, "parameters": 16777216, "reference_delta_l2": 1.2620082729612303, "retained_delta_l2": 0.9977812321989201, "delta_retention_norm_ratio": 0.7906297078843101, "delta_cosine": 0.6084140135379782, "relative_weight_rmse": 0.010243938736757576 }, { "index": 217, "total": 252, "projection": "layers.4.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 1.1348753309421484, "retained_delta_l2": 1.077055790992353, "delta_retention_norm_ratio": 0.9490520779037512, "delta_cosine": 0.7689872270233765, "relative_weight_rmse": 0.013531675501347078 }, { "index": 218, "total": 252, "projection": "layers.5.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4312, "parameters": 50331648, "reference_delta_l2": 1.8065839643892028, "retained_delta_l2": 1.3452533951368082, "delta_retention_norm_ratio": 0.7446392869935784, "delta_cosine": 0.5698075119000315, "relative_weight_rmse": 0.009053654516748385 }, { "index": 219, "total": 252, "projection": "layers.5.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3906, "parameters": 50331648, "reference_delta_l2": 2.9535589675034295, "retained_delta_l2": 2.48962526367793, "delta_retention_norm_ratio": 0.8429238390260916, "delta_cosine": 0.6604189599227185, "relative_weight_rmse": 0.00999517624475973 }, { "index": 220, "total": 252, "projection": "layers.5.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3916, "parameters": 50331648, "reference_delta_l2": 3.1775253397334686, "retained_delta_l2": 2.9633965747586823, "delta_retention_norm_ratio": 0.9326114689638486, "delta_cosine": 0.7494165594063157, "relative_weight_rmse": 0.013127994811945903 }, { "index": 221, "total": 252, "projection": "layers.5.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0311, "parameters": 4194304, "reference_delta_l2": 0.6207896422405921, "retained_delta_l2": 0.48910098674505714, "delta_retention_norm_ratio": 0.7878691161465965, "delta_cosine": 0.6084948994396171, "relative_weight_rmse": 0.00986678113185319 }, { "index": 222, "total": 252, "projection": "layers.5.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.135, "parameters": 16777216, "reference_delta_l2": 1.468287392795666, "retained_delta_l2": 1.2335401666266865, "delta_retention_norm_ratio": 0.8401217450202216, "delta_cosine": 0.6521262975815546, "relative_weight_rmse": 0.01135293861593865 }, { "index": 223, "total": 252, "projection": "layers.5.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1335, "parameters": 16777216, "reference_delta_l2": 1.4697062134668988, "retained_delta_l2": 1.255421403051614, "delta_retention_norm_ratio": 0.8541988810744653, "delta_cosine": 0.6707564193730499, "relative_weight_rmse": 0.011192511327309623 }, { "index": 224, "total": 252, "projection": "layers.5.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0303, "parameters": 4194304, "reference_delta_l2": 1.046720925079732, "retained_delta_l2": 0.9822764761169425, "delta_retention_norm_ratio": 0.9384320620533304, "delta_cosine": 0.7609794098905587, "relative_weight_rmse": 0.012878622273056339 }, { "index": 225, "total": 252, "projection": "layers.6.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4324, "parameters": 50331648, "reference_delta_l2": 4.763581604307177, "retained_delta_l2": 4.776047230176001, "delta_retention_norm_ratio": 1.0026168599394945, "delta_cosine": 0.8440799355337425, "relative_weight_rmse": 0.015429444299361368 }, { "index": 226, "total": 252, "projection": "layers.6.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3916, "parameters": 50331648, "reference_delta_l2": 3.105994770214036, "retained_delta_l2": 2.7156819801891334, "delta_retention_norm_ratio": 0.874335657687535, "delta_cosine": 0.6920577735011633, "relative_weight_rmse": 0.010681939796870762 }, { "index": 227, "total": 252, "projection": "layers.6.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3917, "parameters": 50331648, "reference_delta_l2": 3.385041898713471, "retained_delta_l2": 3.1967324013083513, "delta_retention_norm_ratio": 0.944370113269, "delta_cosine": 0.762999101289802, "relative_weight_rmse": 0.013216400552743588 }, { "index": 228, "total": 252, "projection": "layers.6.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0311, "parameters": 4194304, "reference_delta_l2": 0.5923543402760572, "retained_delta_l2": 0.44722429559841054, "delta_retention_norm_ratio": 0.754994544971155, "delta_cosine": 0.5781239453280363, "relative_weight_rmse": 0.009575510514011771 }, { "index": 229, "total": 252, "projection": "layers.6.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1367, "parameters": 16777216, "reference_delta_l2": 1.3637666672894704, "retained_delta_l2": 1.105231379185307, "delta_retention_norm_ratio": 0.810425570366806, "delta_cosine": 0.6263504572146829, "relative_weight_rmse": 0.01038836662554142 }, { "index": 230, "total": 252, "projection": "layers.6.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1335, "parameters": 16777216, "reference_delta_l2": 1.4025034551702102, "retained_delta_l2": 1.1632515080468524, "delta_retention_norm_ratio": 0.8294107966426921, "delta_cosine": 0.6445231369465194, "relative_weight_rmse": 0.010719962886095127 }, { "index": 231, "total": 252, "projection": "layers.6.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0306, "parameters": 4194304, "reference_delta_l2": 0.9849644445899759, "retained_delta_l2": 0.9089375611654249, "delta_retention_norm_ratio": 0.9228125605526809, "delta_cosine": 0.739246623026045, "relative_weight_rmse": 0.012389586886707311 }, { "index": 232, "total": 252, "projection": "layers.7.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4338, "parameters": 50331648, "reference_delta_l2": 1.845972451357094, "retained_delta_l2": 1.3659906541197822, "delta_retention_norm_ratio": 0.7399843118544667, "delta_cosine": 0.5655852668367456, "relative_weight_rmse": 0.00896342088611992 }, { "index": 233, "total": 252, "projection": "layers.7.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.4004, "parameters": 50331648, "reference_delta_l2": 2.8972880598888335, "retained_delta_l2": 2.4823971140636263, "delta_retention_norm_ratio": 0.8568002431069536, "delta_cosine": 0.6737961212272754, "relative_weight_rmse": 0.010353212347255036 }, { "index": 234, "total": 252, "projection": "layers.7.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3954, "parameters": 50331648, "reference_delta_l2": 2.976464301108795, "retained_delta_l2": 2.693361096432054, "delta_retention_norm_ratio": 0.9048860742017706, "delta_cosine": 0.71663898129702, "relative_weight_rmse": 0.01238607089131992 }, { "index": 235, "total": 252, "projection": "layers.7.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.032, "parameters": 4194304, "reference_delta_l2": 0.7227176135675857, "retained_delta_l2": 0.6135642538998713, "delta_retention_norm_ratio": 0.8489681756489987, "delta_cosine": 0.6626721699461946, "relative_weight_rmse": 0.010772914353297651 }, { "index": 236, "total": 252, "projection": "layers.7.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1364, "parameters": 16777216, "reference_delta_l2": 1.3660572752775417, "retained_delta_l2": 1.1066998580108307, "delta_retention_norm_ratio": 0.8101416229315733, "delta_cosine": 0.6252957723577145, "relative_weight_rmse": 0.010550120700751548 }, { "index": 237, "total": 252, "projection": "layers.7.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1352, "parameters": 16777216, "reference_delta_l2": 1.6868798978306623, "retained_delta_l2": 1.5252113523148896, "delta_retention_norm_ratio": 0.9041611997844782, "delta_cosine": 0.7181098490516066, "relative_weight_rmse": 0.012121673317143276 }, { "index": 238, "total": 252, "projection": "layers.7.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0311, "parameters": 4194304, "reference_delta_l2": 0.9902755720656208, "retained_delta_l2": 0.8939089233714745, "delta_retention_norm_ratio": 0.9026870384239263, "delta_cosine": 0.7179580569495855, "relative_weight_rmse": 0.011989921320523676 }, { "index": 239, "total": 252, "projection": "layers.8.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4369, "parameters": 50331648, "reference_delta_l2": 2.641258801378591, "retained_delta_l2": 2.2794388272454036, "delta_retention_norm_ratio": 0.8630122977936363, "delta_cosine": 0.677297685340227, "relative_weight_rmse": 0.011057474749445264 }, { "index": 240, "total": 252, "projection": "layers.8.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3945, "parameters": 50331648, "reference_delta_l2": 2.2050936875452227, "retained_delta_l2": 1.6625676195063588, "delta_retention_norm_ratio": 0.7539668853513337, "delta_cosine": 0.5773198349692906, "relative_weight_rmse": 0.009475486385452227 }, { "index": 241, "total": 252, "projection": "layers.8.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3945, "parameters": 50331648, "reference_delta_l2": 2.679677747498109, "retained_delta_l2": 2.284396234756145, "delta_retention_norm_ratio": 0.8524891610153421, "delta_cosine": 0.6657822834602609, "relative_weight_rmse": 0.011427666874516884 }, { "index": 242, "total": 252, "projection": "layers.8.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0333, "parameters": 4194304, "reference_delta_l2": 0.8932447143308502, "retained_delta_l2": 0.8098993999824166, "delta_retention_norm_ratio": 0.9066937503113361, "delta_cosine": 0.7197835568554013, "relative_weight_rmse": 0.012127856844388323 }, { "index": 243, "total": 252, "projection": "layers.8.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.1357, "parameters": 16777216, "reference_delta_l2": 1.5718337251376286, "retained_delta_l2": 1.3461363236464665, "delta_retention_norm_ratio": 0.8564114016122156, "delta_cosine": 0.6703790087422654, "relative_weight_rmse": 0.011303805629343487 }, { "index": 244, "total": 252, "projection": "layers.8.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1355, "parameters": 16777216, "reference_delta_l2": 1.5937350677877007, "retained_delta_l2": 1.382939622920773, "delta_retention_norm_ratio": 0.867734952234227, "delta_cosine": 0.679088737874005, "relative_weight_rmse": 0.01183446250094784 }, { "index": 245, "total": 252, "projection": "layers.8.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0312, "parameters": 4194304, "reference_delta_l2": 0.7963146426204377, "retained_delta_l2": 0.6603613999230062, "delta_retention_norm_ratio": 0.829271954299309, "delta_cosine": 0.6434056559316997, "relative_weight_rmse": 0.01062666469671945 }, { "index": 246, "total": 252, "projection": "layers.9.mlp.down_proj", "shape": [ 4096, 12288 ], "seconds": 0.4357, "parameters": 50331648, "reference_delta_l2": 2.467327752515527, "retained_delta_l2": 2.096186873024144, "delta_retention_norm_ratio": 0.8495777956078224, "delta_cosine": 0.6640106694351272, "relative_weight_rmse": 0.010877294060438782 }, { "index": 247, "total": 252, "projection": "layers.9.mlp.gate_proj", "shape": [ 12288, 4096 ], "seconds": 0.3917, "parameters": 50331648, "reference_delta_l2": 2.698803219715491, "retained_delta_l2": 2.210001490704963, "delta_retention_norm_ratio": 0.8188820416991878, "delta_cosine": 0.6350180124903053, "relative_weight_rmse": 0.010370650234180913 }, { "index": 248, "total": 252, "projection": "layers.9.mlp.up_proj", "shape": [ 12288, 4096 ], "seconds": 0.3925, "parameters": 50331648, "reference_delta_l2": 3.0233243335706885, "retained_delta_l2": 2.741480983449664, "delta_retention_norm_ratio": 0.9067770047058915, "delta_cosine": 0.7217231268446289, "relative_weight_rmse": 0.01238018059960264 }, { "index": 249, "total": 252, "projection": "layers.9.self_attn.k_proj", "shape": [ 1024, 4096 ], "seconds": 0.0321, "parameters": 4194304, "reference_delta_l2": 0.8387394521719984, "retained_delta_l2": 0.7567867999749869, "delta_retention_norm_ratio": 0.9022906911260856, "delta_cosine": 0.7183405611602802, "relative_weight_rmse": 0.011787582836933876 }, { "index": 250, "total": 252, "projection": "layers.9.self_attn.o_proj", "shape": [ 4096, 4096 ], "seconds": 0.135, "parameters": 16777216, "reference_delta_l2": 1.5247160233711252, "retained_delta_l2": 1.299308342528706, "delta_retention_norm_ratio": 0.852164155562525, "delta_cosine": 0.6666228983984552, "relative_weight_rmse": 0.011154261210944667 }, { "index": 251, "total": 252, "projection": "layers.9.self_attn.q_proj", "shape": [ 4096, 4096 ], "seconds": 0.1379, "parameters": 16777216, "reference_delta_l2": 1.8040588235553003, "retained_delta_l2": 1.6572263164999084, "delta_retention_norm_ratio": 0.9186099116402281, "delta_cosine": 0.7348702177287972, "relative_weight_rmse": 0.012698203899833869 }, { "index": 252, "total": 252, "projection": "layers.9.self_attn.v_proj", "shape": [ 1024, 4096 ], "seconds": 0.0308, "parameters": 4194304, "reference_delta_l2": 0.8010147536640196, "retained_delta_l2": 0.6707311777466941, "delta_retention_norm_ratio": 0.8373518398738855, "delta_cosine": 0.6534294135001347, "relative_weight_rmse": 0.01042703006403757 } ] }