Auto upload zain 2026-08-12T18:47:12.697939

.gitattributes

@@ -38,3 +38,4 @@ zain/Activation/wandb/run-20260812_180713-utqzlg67/run-utqzlg67.wandb filter=lfs
 zain/Activation/wandb/run-20260812_182126-vyycsg6r/run-vyycsg6r.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260812_183021-5nis3j8m/run-5nis3j8m.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260812_183823-qtstmmvv/run-qtstmmvv.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260812_184213-sjvvelzu/run-sjvvelzu.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/out/glu-relu-75L_run/training_log.jsonl

@@ -0,0 +1,30 @@
+{"step": 20, "epoch": 0.0003370407819346141, "timestamp": 1786560138.123157, "loss": 8.21759033203125, "grad_norm": 1.6796875, "learning_rate": 9.5e-05, "train/total_time_seconds": 3.2094581723213196, "train/time_per_step_avg": 0.16047290861606597, "train/epoch_time_elapsed": 3.549226962029934, "train/estimated_remaining_minutes": 2.6210575073957445}
+{"step": 40, "epoch": 0.0006740815638692282, "timestamp": 1786560141.60977, "loss": 7.848448944091797, "grad_norm": 1.28125, "learning_rate": 0.00019500000000000002, "train/total_time_seconds": 6.412547700107098, "train/time_per_step_avg": 0.16031369250267744, "train/epoch_time_elapsed": 7.035838417708874, "train/estimated_remaining_minutes": 2.565019080042839}
+{"step": 50, "epoch": 0.0008426019548365353, "timestamp": 1786560168.6928885, "eval_loss": 7.331467628479004, "eval_runtime": 25.3435, "eval_samples_per_second": 375.914, "eval_steps_per_second": 11.758, "train/total_time_seconds": 8.00680946558714, "train/time_per_step_avg": 0.16013618931174278, "train/epoch_time_elapsed": 34.118958327919245, "train/estimated_remaining_minutes": 2.535489664102594}
+{"step": 60, "epoch": 0.0010111223458038423, "timestamp": 1786560170.4503865, "loss": 7.344553375244141, "grad_norm": 1.2265625, "learning_rate": 0.000295, "train/total_time_seconds": 9.622875556349754, "train/time_per_step_avg": 0.1603812592724959, "train/epoch_time_elapsed": 35.876456290483475, "train/estimated_remaining_minutes": 2.5126397286024362}
+{"step": 80, "epoch": 0.0013481631277384564, "timestamp": 1786560173.9503767, "loss": 6.737104034423828, "grad_norm": 1.046875, "learning_rate": 0.000395, "train/total_time_seconds": 12.84031467884779, "train/time_per_step_avg": 0.16050393348559738, "train/epoch_time_elapsed": 39.37644646316767, "train/estimated_remaining_minutes": 2.4610603134458264}
+{"step": 100, "epoch": 0.0016852039096730705, "timestamp": 1786560177.455623, "loss": 6.247975921630859, "grad_norm": 0.671875, "learning_rate": 0.000495, "train/total_time_seconds": 16.05445696413517, "train/time_per_step_avg": 0.1605445696413517, "train/epoch_time_elapsed": 42.881691846996546, "train/estimated_remaining_minutes": 2.4081685446202754}
+{"step": 100, "epoch": 0.0016852039096730705, "timestamp": 1786560202.5107224, "eval_loss": 6.0784220695495605, "eval_runtime": 25.0534, "eval_samples_per_second": 380.268, "eval_steps_per_second": 11.895, "train/total_time_seconds": 16.05445696413517, "train/time_per_step_avg": 0.1605445696413517, "train/epoch_time_elapsed": 67.93679214641452, "train/estimated_remaining_minutes": 2.4081685446202754}
+{"step": 120, "epoch": 0.0020222446916076846, "timestamp": 1786560205.9211605, "loss": 5.984897232055664, "grad_norm": 1.8125, "learning_rate": 0.0005949999999999999, "train/total_time_seconds": 19.195178482681513, "train/time_per_step_avg": 0.15985720310360194, "train/epoch_time_elapsed": 71.34723086282611, "train/estimated_remaining_minutes": 2.346077370105518}
+{"step": 140, "epoch": 0.0023592854735422987, "timestamp": 1786560209.3643422, "loss": 5.7836250305175785, "grad_norm": 0.5234375, "learning_rate": 0.000695, "train/total_time_seconds": 22.369587503373623, "train/time_per_step_avg": 0.15957039803266526, "train/epoch_time_elapsed": 74.79041273519397, "train/estimated_remaining_minutes": 2.290219672964442}
+{"step": 150, "epoch": 0.0025278058645096056, "timestamp": 1786560236.9964783, "eval_loss": 5.560763359069824, "eval_runtime": 25.9025, "eval_samples_per_second": 367.802, "eval_steps_per_second": 11.505, "train/total_time_seconds": 23.961229860782623, "train/time_per_step_avg": 0.15954420395195484, "train/epoch_time_elapsed": 102.42254745587707, "train/estimated_remaining_minutes": 2.263005042407248}
+{"step": 160, "epoch": 0.002696326255476913, "timestamp": 1786560238.7289863, "loss": 5.57129135131836, "grad_norm": 0.83203125, "learning_rate": 0.000795, "train/total_time_seconds": 25.55166833847761, "train/time_per_step_avg": 0.15928792782127857, "train/epoch_time_elapsed": 104.15505620092154, "train/estimated_remaining_minutes": 2.235770979616791}
+{"step": 180, "epoch": 0.003033367037411527, "timestamp": 1786560242.1886723, "loss": 5.3551380157470705, "grad_norm": 1.6015625, "learning_rate": 0.0008950000000000001, "train/total_time_seconds": 28.740631844848394, "train/time_per_step_avg": 0.15900317166000605, "train/epoch_time_elapsed": 107.61474177613854, "train/estimated_remaining_minutes": 2.182159084516267}
+{"step": 200, "epoch": 0.003370407819346141, "timestamp": 1786560245.802692, "loss": 5.140692901611328, "grad_norm": 0.53515625, "learning_rate": 0.000995, "train/total_time_seconds": 32.070018671453, "train/time_per_step_avg": 0.1601556170731783, "train/epoch_time_elapsed": 111.22876178473234, "train/estimated_remaining_minutes": 2.1380012447635335}
+{"step": 200, "epoch": 0.003370407819346141, "timestamp": 1786560270.5646892, "eval_loss": 4.980733394622803, "eval_runtime": 24.7605, "eval_samples_per_second": 384.766, "eval_steps_per_second": 12.035, "train/total_time_seconds": 32.070018671453, "train/time_per_step_avg": 0.1601556170731783, "train/epoch_time_elapsed": 135.9907580204308, "train/estimated_remaining_minutes": 2.1380012447635335}
+{"step": 220, "epoch": 0.003707448601280755, "timestamp": 1786560273.9519782, "loss": 4.899646759033203, "grad_norm": 0.60546875, "learning_rate": 0.001, "train/total_time_seconds": 35.18007303029299, "train/time_per_step_avg": 0.15984894547611475, "train/epoch_time_elapsed": 139.37804751470685, "train/estimated_remaining_minutes": 2.0788224972445857}
+{"step": 240, "epoch": 0.004044489383215369, "timestamp": 1786560277.4466531, "loss": 4.673993682861328, "grad_norm": 0.58984375, "learning_rate": 0.001, "train/total_time_seconds": 38.38174121454358, "train/time_per_step_avg": 0.16012153711169957, "train/epoch_time_elapsed": 142.87272255122662, "train/estimated_remaining_minutes": 2.0257030085453556}
+{"step": 250, "epoch": 0.0042130097741826765, "timestamp": 1786560304.9413152, "eval_loss": 4.473351001739502, "eval_runtime": 25.7715, "eval_samples_per_second": 369.672, "eval_steps_per_second": 11.563, "train/total_time_seconds": 39.96863412857056, "train/time_per_step_avg": 0.16007404267787934, "train/epoch_time_elapsed": 170.36738376319408, "train/estimated_remaining_minutes": 1.998431706428528}
+{"step": 260, "epoch": 0.004381530165149983, "timestamp": 1786560306.7209792, "loss": 4.466047668457032, "grad_norm": 0.78515625, "learning_rate": 0.001, "train/total_time_seconds": 41.59740253165364, "train/time_per_step_avg": 0.1604573419317603, "train/epoch_time_elapsed": 172.1470484957099, "train/estimated_remaining_minutes": 1.9732101200912626}
+{"step": 280, "epoch": 0.0047185709470845974, "timestamp": 1786560310.2039845, "loss": 4.300079727172852, "grad_norm": 0.76171875, "learning_rate": 0.001, "train/total_time_seconds": 44.801249749958515, "train/time_per_step_avg": 0.1606061790511012, "train/epoch_time_elapsed": 175.63005308806896, "train/estimated_remaining_minutes": 1.9200535607125078}
+{"step": 300, "epoch": 0.005055611729019211, "timestamp": 1786560313.680988, "loss": 4.177681350708008, "grad_norm": 0.73046875, "learning_rate": 0.001, "train/total_time_seconds": 47.99559833481908, "train/time_per_step_avg": 0.1592557966336608, "train/epoch_time_elapsed": 179.1070576272905, "train/estimated_remaining_minutes": 1.8664954907985196}
+{"step": 300, "epoch": 0.005055611729019211, "timestamp": 1786560338.542324, "eval_loss": 4.133443832397461, "eval_runtime": 24.8597, "eval_samples_per_second": 383.231, "eval_steps_per_second": 11.987, "train/total_time_seconds": 47.99559833481908, "train/time_per_step_avg": 0.1592557966336608, "train/epoch_time_elapsed": 203.96839321032166, "train/estimated_remaining_minutes": 1.8664954907985196}
+{"step": 320, "epoch": 0.005392652510953826, "timestamp": 1786560342.0987465, "loss": 4.078357696533203, "grad_norm": 0.63671875, "learning_rate": 0.001, "train/total_time_seconds": 51.2690104842186, "train/time_per_step_avg": 0.16088937453925609, "train/epoch_time_elapsed": 207.52481628209352, "train/estimated_remaining_minutes": 1.8157774546494088}
+{"step": 340, "epoch": 0.005729693292888439, "timestamp": 1786560345.5869186, "loss": 3.9902496337890625, "grad_norm": 0.458984375, "learning_rate": 0.001, "train/total_time_seconds": 54.48344559967518, "train/time_per_step_avg": 0.161017043851316, "train/epoch_time_elapsed": 211.0129889138043, "train/estimated_remaining_minutes": 1.7626997105777265}
+{"step": 350, "epoch": 0.005898213683855747, "timestamp": 1786560373.0741432, "eval_loss": 3.9098942279815674, "eval_runtime": 25.7688, "eval_samples_per_second": 369.71, "eval_steps_per_second": 11.564, "train/total_time_seconds": 56.06589589640498, "train/time_per_step_avg": 0.16097261767834425, "train/epoch_time_elapsed": 238.50021233782172, "train/estimated_remaining_minutes": 1.7353729682220589}
+{"step": 360, "epoch": 0.006066734074823054, "timestamp": 1786560374.7806027, "loss": 3.927245330810547, "grad_norm": 0.7265625, "learning_rate": 0.001, "train/total_time_seconds": 57.6324389539659, "train/time_per_step_avg": 0.1603503642231226, "train/epoch_time_elapsed": 240.20667296275496, "train/estimated_remaining_minutes": 1.7076278208582492}
+{"step": 380, "epoch": 0.0064037748567576675, "timestamp": 1786560378.2182276, "loss": 3.831654357910156, "grad_norm": 0.44921875, "learning_rate": 0.001, "train/total_time_seconds": 60.79468631744385, "train/time_per_step_avg": 0.15993436567485333, "train/epoch_time_elapsed": 243.64429780095816, "train/estimated_remaining_minutes": 1.6531888384568063}
+{"step": 400, "epoch": 0.006740815638692282, "timestamp": 1786560381.6353347, "loss": 3.7726951599121095, "grad_norm": 0.50390625, "learning_rate": 0.001, "train/total_time_seconds": 63.93965522944927, "train/time_per_step_avg": 0.15944056894630193, "train/epoch_time_elapsed": 247.06140458211303, "train/estimated_remaining_minutes": 1.598491380736232}
+{"step": 400, "epoch": 0.006740815638692282, "timestamp": 1786560406.7175198, "eval_loss": 3.7514989376068115, "eval_runtime": 25.0804, "eval_samples_per_second": 379.859, "eval_steps_per_second": 11.882, "train/total_time_seconds": 63.93965522944927, "train/time_per_step_avg": 0.15944056894630193, "train/epoch_time_elapsed": 272.1435889005661, "train/estimated_remaining_minutes": 1.598491380736232}
+{"step": 420, "epoch": 0.007077856420626896, "timestamp": 1786560410.28777, "loss": 3.7323013305664063, "grad_norm": 0.458984375, "learning_rate": 0.001, "train/total_time_seconds": 67.22942125052214, "train/time_per_step_avg": 0.1596041076630354, "train/epoch_time_elapsed": 275.71384005993605, "train/estimated_remaining_minutes": 1.5473438224326523}
+{"step": 440, "epoch": 0.00741489720256151, "timestamp": 1786560413.8173282, "loss": 3.6785472869873046, "grad_norm": 0.625, "learning_rate": 0.001, "train/total_time_seconds": 70.47488746792078, "train/time_per_step_avg": 0.159914418682456, "train/epoch_time_elapsed": 279.2433968856931, "train/estimated_remaining_minutes": 1.4949218553801378}

zain/Activation/out/glu-sigmoid-150L_run/training_log.jsonl

@@ -11,3 +11,20 @@
 {"step": 160, "epoch": 0.010785305021907651, "timestamp": 1786560097.858309, "loss": 5.8792579650878904, "grad_norm": 0.365234375, "learning_rate": 0.003975, "train/total_time_seconds": 92.4144375063479, "train/time_per_step_avg": 0.5779887476563453, "train/epoch_time_elapsed": 194.07505892962217, "train/estimated_remaining_minutes": 8.086263281805442}
 {"step": 180, "epoch": 0.012133468149646108, "timestamp": 1786560110.4396544, "loss": 5.723233413696289, "grad_norm": 0.59765625, "learning_rate": 0.004475, "train/total_time_seconds": 104.01449612900615, "train/time_per_step_avg": 0.5769760990515351, "train/epoch_time_elapsed": 206.65640380606055, "train/estimated_remaining_minutes": 7.897396928313429}
 {"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786560122.678806, "loss": 5.667977523803711, "grad_norm": 0.296875, "learning_rate": 0.004975, "train/total_time_seconds": 115.22981420531869, "train/time_per_step_avg": 0.5762379133701324, "train/epoch_time_elapsed": 218.89555491134524, "train/estimated_remaining_minutes": 7.6819876136879115}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786560152.6929605, "eval_loss": 5.4667558670043945, "eval_runtime": 30.0125, "eval_samples_per_second": 317.434, "eval_steps_per_second": 2.499, "train/total_time_seconds": 115.22981420531869, "train/time_per_step_avg": 0.5762379133701324, "train/epoch_time_elapsed": 248.90970926731825, "train/estimated_remaining_minutes": 7.6819876136879115}
+{"step": 220, "epoch": 0.01482979440512302, "timestamp": 1786560164.9443223, "loss": 5.387565612792969, "grad_norm": 0.384765625, "learning_rate": 0.005, "train/total_time_seconds": 126.49688804149628, "train/time_per_step_avg": 0.5734840777516365, "train/epoch_time_elapsed": 261.1610717512667, "train/estimated_remaining_minutes": 7.474816111542961}
+{"step": 240, "epoch": 0.016177957532861477, "timestamp": 1786560177.736363, "loss": 5.223324584960937, "grad_norm": 0.181640625, "learning_rate": 0.005, "train/total_time_seconds": 138.26898029074073, "train/time_per_step_avg": 0.5719823825731873, "train/epoch_time_elapsed": 273.95311187952757, "train/estimated_remaining_minutes": 7.2975295153446496}
+{"step": 250, "epoch": 0.016852039096730706, "timestamp": 1786560214.7755182, "eval_loss": 5.092136383056641, "eval_runtime": 31.0263, "eval_samples_per_second": 307.062, "eval_steps_per_second": 2.417, "train/total_time_seconds": 143.79492635652423, "train/time_per_step_avg": 0.5708510028198361, "train/epoch_time_elapsed": 310.99226642400026, "train/estimated_remaining_minutes": 7.18974631782621}
+{"step": 260, "epoch": 0.01752612066059993, "timestamp": 1786560220.9513483, "loss": 5.0866455078125, "grad_norm": 0.58203125, "learning_rate": 0.005, "train/total_time_seconds": 149.48124688491225, "train/time_per_step_avg": 0.5706680937856435, "train/epoch_time_elapsed": 317.1680973879993, "train/estimated_remaining_minutes": 7.0907770958227605}
+{"step": 280, "epoch": 0.01887428378833839, "timestamp": 1786560233.4454212, "loss": 5.2055717468261715, "grad_norm": 1.609375, "learning_rate": 0.005, "train/total_time_seconds": 161.00160675123334, "train/time_per_step_avg": 0.5698711062222719, "train/epoch_time_elapsed": 329.6621695756912, "train/estimated_remaining_minutes": 6.900068860767143}
+{"step": 300, "epoch": 0.020222446916076844, "timestamp": 1786560246.3871453, "loss": 5.605699920654297, "grad_norm": 2.078125, "learning_rate": 0.005, "train/total_time_seconds": 172.92131251841784, "train/time_per_step_avg": 0.5769149831309914, "train/epoch_time_elapsed": 342.60389437153935, "train/estimated_remaining_minutes": 6.724717709049583}
+{"step": 300, "epoch": 0.020222446916076844, "timestamp": 1786560277.2816527, "eval_loss": 5.3450422286987305, "eval_runtime": 30.893, "eval_samples_per_second": 308.387, "eval_steps_per_second": 2.428, "train/total_time_seconds": 172.92131251841784, "train/time_per_step_avg": 0.5769149831309914, "train/epoch_time_elapsed": 373.49840097129345, "train/estimated_remaining_minutes": 6.724717709049583}
+{"step": 320, "epoch": 0.021570610043815303, "timestamp": 1786560289.6595883, "loss": 5.227577972412109, "grad_norm": 512.0, "learning_rate": 0.005, "train/total_time_seconds": 184.29014735668898, "train/time_per_step_avg": 0.577932593151927, "train/epoch_time_elapsed": 385.8763385117054, "train/estimated_remaining_minutes": 6.526942718882735}
+{"step": 340, "epoch": 0.022918773171553757, "timestamp": 1786560302.132462, "loss": 5.299805450439453, "grad_norm": 2.0625, "learning_rate": 0.005, "train/total_time_seconds": 195.69051584228873, "train/time_per_step_avg": 0.5742153555154801, "train/epoch_time_elapsed": 398.3492114506662, "train/estimated_remaining_minutes": 6.331163747838754}
+{"step": 350, "epoch": 0.023592854735422986, "timestamp": 1786560339.2187812, "eval_loss": 5.322297096252441, "eval_runtime": 30.74, "eval_samples_per_second": 309.922, "eval_steps_per_second": 2.44, "train/total_time_seconds": 201.55185126140714, "train/time_per_step_avg": 0.5775692490488291, "train/epoch_time_elapsed": 435.4355287961662, "train/estimated_remaining_minutes": 6.238509681900697}
+{"step": 360, "epoch": 0.024266936299292215, "timestamp": 1786560345.7392898, "loss": 5.430966186523437, "grad_norm": 3.078125, "learning_rate": 0.005, "train/total_time_seconds": 207.57690269872546, "train/time_per_step_avg": 0.5809565581381321, "train/epoch_time_elapsed": 441.9560389146209, "train/estimated_remaining_minutes": 6.150426746628901}
+{"step": 380, "epoch": 0.02561509942703067, "timestamp": 1786560358.151074, "loss": 5.5070240020751955, "grad_norm": 3.140625, "learning_rate": 0.005, "train/total_time_seconds": 219.0110193863511, "train/time_per_step_avg": 0.5800941263511777, "train/epoch_time_elapsed": 454.36782310530543, "train/estimated_remaining_minutes": 5.95556280787446}
+{"step": 400, "epoch": 0.026963262554769128, "timestamp": 1786560370.628514, "loss": 5.358203887939453, "grad_norm": 0.23046875, "learning_rate": 0.005, "train/total_time_seconds": 230.48501763492823, "train/time_per_step_avg": 0.5756370511651039, "train/epoch_time_elapsed": 466.84526336193085, "train/estimated_remaining_minutes": 5.762125440873206}
+{"step": 400, "epoch": 0.026963262554769128, "timestamp": 1786560401.5902734, "eval_loss": 5.298312664031982, "eval_runtime": 30.9601, "eval_samples_per_second": 307.719, "eval_steps_per_second": 2.422, "train/total_time_seconds": 230.48501763492823, "train/time_per_step_avg": 0.5756370511651039, "train/epoch_time_elapsed": 497.8070211522281, "train/estimated_remaining_minutes": 5.762125440873206}
+{"step": 420, "epoch": 0.028311425682507583, "timestamp": 1786560414.09902, "loss": 5.231809997558594, "grad_norm": 0.65234375, "learning_rate": 0.005, "train/total_time_seconds": 242.0218550786376, "train/time_per_step_avg": 0.5773170772194862, "train/epoch_time_elapsed": 510.31576934829354, "train/estimated_remaining_minutes": 5.5703442835559445}
+{"step": 440, "epoch": 0.02965958881024604, "timestamp": 1786560426.4843917, "loss": 5.145277786254883, "grad_norm": 0.84375, "learning_rate": 0.005, "train/total_time_seconds": 253.40580972656608, "train/time_per_step_avg": 0.5771529388427734, "train/epoch_time_elapsed": 522.7011404894292, "train/estimated_remaining_minutes": 5.3752747517756445}

zain/Activation/out/glu-silu-75L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 75,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-75L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8779561423a0618884138c20f4e68e63c83dba3cd4b6bbfcb49cff5f5cca91d5
+size 25737568

zain/Activation/out/glu-silu-75L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-75L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:954b9896ded9312046b3a63252dc6a327ab94f4db2083972bb5b01198e71fda8
+size 4920

zain/Activation/out/glu-silu-75L_run/training_log.jsonl

@@ -69,3 +69,4 @@
 {"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560081.5580728, "loss": 3.05130615234375, "grad_norm": 0.59765625, "learning_rate": 0.001, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 659.760448589921, "train/estimated_remaining_minutes": 0.0}
 {"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560106.9211233, "eval_loss": 3.045870542526245, "eval_runtime": 25.3615, "eval_samples_per_second": 375.648, "eval_steps_per_second": 11.75, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 685.1234988421202, "train/estimated_remaining_minutes": 0.0}
 {"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560107.1479647, "train_runtime": 686.5633, "train_samples_per_second": 23.304, "train_steps_per_second": 1.457, "total_flos": 604929785856000.0, "train_loss": 4.216384292602539, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 685.3503403961658, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560132.4857523, "eval_loss": 3.045870542526245, "eval_runtime": 25.3343, "eval_samples_per_second": 376.051, "eval_steps_per_second": 11.763, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 710.6881274692714, "train/estimated_remaining_minutes": 0.0}

zain/Activation/wandb/debug-internal.log

@@ -1,39 +1,47 @@
-{"time":"2026-08-12T18:38:23.105010622Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-12T18:38:23.105150417Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-12T18:38:23.371916432Z","level":"INFO","msg":"stream: created new stream","id":"qtstmmvv"}
-{"time":"2026-08-12T18:38:23.372014762Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-12T18:38:23.372116804Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-12T18:38:23.372128482Z","level":"INFO","msg":"writer: started","stream_id":"qtstmmvv"}
-{"time":"2026-08-12T18:38:23.372151282Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-12T18:38:24.743864927Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:38:24.838824501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:38:39.744346348Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
-{"time":"2026-08-12T18:38:39.869953996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:38:54.744630635Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:38:54.899949077Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:39:09.744851347Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:39:09.862852929Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:39:24.744295359Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":2,"console_lines":2}
-{"time":"2026-08-12T18:39:24.889674817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:39:39.744771093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:39:39.90389517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:39:54.744830696Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:39:54.85572055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:40:09.74451592Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:40:09.918048601Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:40:24.744095296Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":3,"console_lines":5}
-{"time":"2026-08-12T18:40:24.871071918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:40:39.744522438Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:40:39.878594588Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:40:54.744332246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":18,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:40:54.86474185Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:41:09.744339141Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:41:09.862772827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:41:24.744324362Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-12T18:41:24.869002616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:41:39.744710255Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":2,"events_offset":24,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:41:39.854076154Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:41:54.744214155Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:41:54.878519291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:42:09.744721347Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:42:09.875728592Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:13.959580652Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T18:42:13.959724138Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T18:42:14.218870288Z","level":"INFO","msg":"stream: created new stream","id":"sjvvelzu"}
+{"time":"2026-08-12T18:42:14.218956754Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T18:42:14.219053701Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T18:42:14.219069228Z","level":"INFO","msg":"writer: started","stream_id":"sjvvelzu"}
+{"time":"2026-08-12T18:42:14.21909666Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T18:42:14.876923676Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:42:14.98880652Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:29.877415684Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":4,"uploaded_len":2}
+{"time":"2026-08-12T18:42:30.005224107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:44.877960291Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:42:44.996668849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:59.877785962Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":4,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:00.000525353Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:14.877602252Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":3,"console_lines":5}
+{"time":"2026-08-12T18:43:15.034494032Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:29.877792727Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":3,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:29.992983921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:44.877553323Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:45.010375305Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:59.877295703Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":2,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:59.994845826Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:14.877645033Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":2,"events_offset":14,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:15.032619695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:29.877901466Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:44:30.016442098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:44.877582443Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":3,"events_offset":18,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:44.989590389Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:59.877434616Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:45:00.021745309Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:14.877672035Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":4,"events_offset":22,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:14.996134045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:29.877315928Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:45:30.002914153Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:44.877777926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":26,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:45.003620078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:59.877083809Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:59.998353063Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:14.877804457Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":2,"events_offset":30,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:15.035374034Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:29.87719442Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":2,"events_offset":32,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:29.987619741Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:44.877017477Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:46:45.008164821Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:59.87728567Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":3,"events_offset":36,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:47:00.024895002Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/debug.log

@@ -1,20 +1,19 @@
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug.log
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-internal.log
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug.log
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug-internal.log
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():772] calling init triggers
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
-2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
-2026-08-12 18:38:23,372 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
-2026-08-12 18:38:23,374 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
-2026-08-12 18:38:23,380 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-12 18:38:23,694 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
-2026-08-12 18:38:23,774 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-12 18:38:23,774 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-12 18:38:23,776 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'sigmoid', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-sigmoid-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-sigmoid-150L-25.1M-20260812-183822', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-sigmoid-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-12 18:38:23,782 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c5940d9050>>
-2026-08-12 18:38:23,783 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():820] starting backend
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 18:42:14,219 INFO    MainThread:1728794 [wandb_init.py:init():840] backend started and connected
+2026-08-12 18:42:14,220 INFO    MainThread:1728794 [wandb_init.py:init():910] updated telemetry
+2026-08-12 18:42:14,227 INFO    MainThread:1728794 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 18:42:14,491 INFO    MainThread:1728794 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 18:42:14,567 INFO    MainThread:1728794 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 18:42:14,568 INFO    MainThread:1728794 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 18:42:14,568 INFO    MainThread:1728794 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 18:42:14,568 INFO    MainThread:1728794 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 18:42:14,569 INFO    MainThread:1728794 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 18:42:14,570 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 75, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'relu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-relu-75L_run', 'per_device_train_batch_size': 16, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-relu-75L-12.8M-20260812-184213', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-relu-75L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 18:42:14,573 INFO    MainThread:1728794 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 12831616 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1523a1b02e90>>
+2026-08-12 18:42:14,573 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 12831616 None

zain/Activation/wandb/run-20260812_183021-5nis3j8m/files/config.yaml

@@ -0,0 +1,445 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            h5f2p7dr96bppkk01acnwchkmgtibv5u:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline75l-halfdata.yaml
+                    - --variants
+                    - glu-silu
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-linear
+                    - glu-waleed10
+                    - mlp-waleed10
+                    - glu-silu-waleed10
+                    - mlp-silu-waleed10
+                    - glu-situglu
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "716186517504"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: c53713bcba08d1849136dad91288c4b82040634e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-12T18:30:21.012951Z"
+                writerId: h5f2p7dr96bppkk01acnwchkmgtibv5u
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: silu
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 50
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-silu-75L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 0
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 12831616
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 75
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-silu-75L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 32
+per_device_train_batch_size:
+    value: 16
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-silu-75L-12.8M-20260812-183019
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260812_183021-5nis3j8m/files/output.log

@@ -77,4 +77,9 @@
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.20it/s]
 100%|██████████| 1000/1000 [11:26<00:00,  1.46it/s], ?it/s]
 {'train_runtime': '686.6', 'train_samples_per_second': '23.3', 'train_steps_per_second': '1.457', 'train_loss': '4.216', 'epoch': '0.01685', 'train/total_time_seconds': '162.5', 'train/time_per_step_avg': '0.1595', 'train/epoch_time_elapsed': '685.4', 'train/estimated_remaining_minutes': '0'}
- 78%|███████▊  | 231/298 [00:19<00:05, 11.75it/s]
+100%|██████████| 298/298 [00:25<00:00, 11.81it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 23.81it/s]

zain/Activation/wandb/run-20260812_183021-5nis3j8m/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/steps_per_second":11.763,"train_samples_per_second":23.304,"_runtime":710,"train/epoch":0.016852039096730706,"train/global_step":1000,"eval/samples_per_second":376.051,"total_flos":6.04929785856e+14,"_timestamp":1.7865601324859946e+09,"eval/runtime":25.3343,"train/train/epoch_time_elapsed":710.6881274692714,"train/train/time_per_step_avg":0.1595082154497504,"_wandb":{"runtime":710},"eval/loss":3.045870542526245,"train_steps_per_second":1.457,"train/grad_norm":0.59765625,"train/train/estimated_remaining_minutes":0,"train_loss":4.216384292602539,"train/loss":3.05130615234375,"train/train/total_time_seconds":162.52781854197383,"_step":71,"train_runtime":686.5633,"train/learning_rate":0.001}

zain/Activation/wandb/run-20260812_183021-5nis3j8m/logs/debug-core.log

@@ -5,3 +5,10 @@
 {"time":"2026-08-12T18:30:21.015604633Z","level":"INFO","msg":"handleInformInit: received","streamId":"5nis3j8m","id":"1(@)"}
 {"time":"2026-08-12T18:30:21.278746041Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"5nis3j8m","id":"1(@)"}
 {"time":"2026-08-12T18:30:26.79263712Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6en7qisn6akl"}
+{"time":"2026-08-12T18:42:12.551682704Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6en7qisn6akl"}
+{"time":"2026-08-12T18:42:13.174674174Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6en7qisn6akl"}
+{"time":"2026-08-12T18:42:13.176324289Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"5nis3j8m","id":"1(@)"}
+{"time":"2026-08-12T18:42:13.177216893Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"5nis3j8m","id":"1(@)"}
+{"time":"2026-08-12T18:42:13.959420776Z","level":"INFO","msg":"handleInformInit: received","streamId":"sjvvelzu","id":"1(@)"}
+{"time":"2026-08-12T18:42:14.219066208Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"sjvvelzu","id":"1(@)"}
+{"time":"2026-08-12T18:42:19.571080912Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8e1ojla26cxt"}

zain/Activation/wandb/run-20260812_183021-5nis3j8m/logs/debug-internal.log

@@ -101,3 +101,11 @@
 {"time":"2026-08-12T18:41:52.059907127Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T18:42:06.911175834Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":91,"events_lines":2,"console_offset":79,"console_lines":1}
 {"time":"2026-08-12T18:42:07.087160098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:12.998681246Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-12T18:42:12.998932217Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":71,"history_lines":1,"events_offset":93,"events_lines":1,"console_offset":79,"console_lines":6,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-12T18:42:13.171972614Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:13.173189206Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-12T18:42:13.176353466Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-12T18:42:13.176367992Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-12T18:42:13.176416544Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-12T18:42:13.176420262Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260812_183021-5nis3j8m/logs/debug.log

@@ -20,3 +20,8 @@ config: {'_wandb': {}}
 2026-08-12 18:30:21,793 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 75, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-75L_run', 'per_device_train_batch_size': 16, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-75L-12.8M-20260812-183019', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-silu-75L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-12 18:30:21,797 INFO    MainThread:1728794 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 12831616 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1523cf1900d0>>
 2026-08-12 18:30:21,797 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 12831616 None
+2026-08-12 18:42:12,550 INFO    MainThread:1728794 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research3/5nis3j8m
+2026-08-12 18:42:12,551 INFO    MainThread:1728794 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-12 18:42:12,551 INFO    MainThread:1728794 [wandb_run.py:_restore():2570] restore
+2026-08-12 18:42:12,551 INFO    MainThread:1728794 [wandb_run.py:_restore():2576] restore done
+2026-08-12 18:42:13,175 INFO    MainThread:1728794 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260812_183021-5nis3j8m/run-5nis3j8m.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a950a00b950559e2a29e05461ef1e292bef6e43a390c0f6f573164f8f4f9ebde
-size 1114112
+oid sha256:bcac4aafc7528821874a9e7a598b5be94fc54b0bb87b9c25db9ad80bdc18c483
+size 1148394

zain/Activation/wandb/run-20260812_183823-qtstmmvv/files/output.log

@@ -1,7 +1,7 @@
- 20%|██        | 200/1000 [03:39<08:01,  1.66it/s]  
+ 44%|████▍     | 445/1000 [08:46<05:42,  1.62it/s]  
 {'loss': '7.948', 'grad_norm': '1.25', 'learning_rate': '0.000475', 'epoch': '0.001348', 'train/total_time_seconds': '11.96', 'train/time_per_step_avg': '0.5978', 'train/epoch_time_elapsed': '13.05', 'train/estimated_remaining_minutes': '9.764'}
 {'loss': '6.824', 'grad_norm': '0.8477', 'learning_rate': '0.000975', 'epoch': '0.002696', 'train/total_time_seconds': '23.24', 'train/time_per_step_avg': '0.5809', 'train/epoch_time_elapsed': '25.33', 'train/estimated_remaining_minutes': '9.295'}
- 23%|██▎       | 17/75 [00:06<00:24,  2.37it/s]
+                                               
 {'eval_loss': '6.051', 'eval_runtime': '31.06', 'eval_samples_per_second': '306.7', 'eval_steps_per_second': '2.414', 'epoch': '0.00337', 'train/total_time_seconds': '28.86', 'train/time_per_step_avg': '0.5772', 'train/epoch_time_elapsed': '62.52', 'train/estimated_remaining_minutes': '9.139'}
 {'loss': '6.1', 'grad_norm': '0.2041', 'learning_rate': '0.001475', 'epoch': '0.004044', 'train/total_time_seconds': '34.62', 'train/time_per_step_avg': '0.5769', 'train/epoch_time_elapsed': '68.76', 'train/estimated_remaining_minutes': '9.039'}
 {'loss': '6.024', 'grad_norm': '0.09668', 'learning_rate': '0.001975', 'epoch': '0.005393', 'train/total_time_seconds': '46.32', 'train/time_per_step_avg': '0.579', 'train/epoch_time_elapsed': '81.44', 'train/estimated_remaining_minutes': '8.877'}
@@ -13,3 +13,20 @@
 {'loss': '5.879', 'grad_norm': '0.3652', 'learning_rate': '0.003975', 'epoch': '0.01079', 'train/total_time_seconds': '92.41', 'train/time_per_step_avg': '0.578', 'train/epoch_time_elapsed': '194.1', 'train/estimated_remaining_minutes': '8.086'}
 {'loss': '5.723', 'grad_norm': '0.5977', 'learning_rate': '0.004475', 'epoch': '0.01213', 'train/total_time_seconds': '104', 'train/time_per_step_avg': '0.577', 'train/epoch_time_elapsed': '206.7', 'train/estimated_remaining_minutes': '7.897'}
 {'loss': '5.668', 'grad_norm': '0.2969', 'learning_rate': '0.004975', 'epoch': '0.01348', 'train/total_time_seconds': '115.2', 'train/time_per_step_avg': '0.5762', 'train/epoch_time_elapsed': '218.9', 'train/estimated_remaining_minutes': '7.682'}
+{'eval_loss': '5.467', 'eval_runtime': '30.01', 'eval_samples_per_second': '317.4', 'eval_steps_per_second': '2.499', 'epoch': '0.01348', 'train/total_time_seconds': '115.2', 'train/time_per_step_avg': '0.5762', 'train/epoch_time_elapsed': '248.9', 'train/estimated_remaining_minutes': '7.682'}
+{'loss': '5.388', 'grad_norm': '0.3848', 'learning_rate': '0.005', 'epoch': '0.01483', 'train/total_time_seconds': '126.5', 'train/time_per_step_avg': '0.5735', 'train/epoch_time_elapsed': '261.2', 'train/estimated_remaining_minutes': '7.475'}
+{'loss': '5.223', 'grad_norm': '0.1816', 'learning_rate': '0.005', 'epoch': '0.01618', 'train/total_time_seconds': '138.3', 'train/time_per_step_avg': '0.572', 'train/epoch_time_elapsed': '274', 'train/estimated_remaining_minutes': '7.298'}
+{'eval_loss': '5.092', 'eval_runtime': '31.03', 'eval_samples_per_second': '307.1', 'eval_steps_per_second': '2.417', 'epoch': '0.01685', 'train/total_time_seconds': '143.8', 'train/time_per_step_avg': '0.5709', 'train/epoch_time_elapsed': '311', 'train/estimated_remaining_minutes': '7.19'}
+{'loss': '5.087', 'grad_norm': '0.582', 'learning_rate': '0.005', 'epoch': '0.01753', 'train/total_time_seconds': '149.5', 'train/time_per_step_avg': '0.5707', 'train/epoch_time_elapsed': '317.2', 'train/estimated_remaining_minutes': '7.091'}
+{'loss': '5.206', 'grad_norm': '1.609', 'learning_rate': '0.005', 'epoch': '0.01887', 'train/total_time_seconds': '161', 'train/time_per_step_avg': '0.5699', 'train/epoch_time_elapsed': '329.7', 'train/estimated_remaining_minutes': '6.9'}
+{'loss': '5.606', 'grad_norm': '2.078', 'learning_rate': '0.005', 'epoch': '0.02022', 'train/total_time_seconds': '172.9', 'train/time_per_step_avg': '0.5769', 'train/epoch_time_elapsed': '342.6', 'train/estimated_remaining_minutes': '6.725'}
+{'eval_loss': '5.345', 'eval_runtime': '30.89', 'eval_samples_per_second': '308.4', 'eval_steps_per_second': '2.428', 'epoch': '0.02022', 'train/total_time_seconds': '172.9', 'train/time_per_step_avg': '0.5769', 'train/epoch_time_elapsed': '373.5', 'train/estimated_remaining_minutes': '6.725'}
+{'loss': '5.228', 'grad_norm': '512', 'learning_rate': '0.005', 'epoch': '0.02157', 'train/total_time_seconds': '184.3', 'train/time_per_step_avg': '0.5779', 'train/epoch_time_elapsed': '385.9', 'train/estimated_remaining_minutes': '6.527'}
+{'loss': '5.3', 'grad_norm': '2.062', 'learning_rate': '0.005', 'epoch': '0.02292', 'train/total_time_seconds': '195.7', 'train/time_per_step_avg': '0.5742', 'train/epoch_time_elapsed': '398.3', 'train/estimated_remaining_minutes': '6.331'}
+{'eval_loss': '5.322', 'eval_runtime': '30.74', 'eval_samples_per_second': '309.9', 'eval_steps_per_second': '2.44', 'epoch': '0.02359', 'train/total_time_seconds': '201.6', 'train/time_per_step_avg': '0.5776', 'train/epoch_time_elapsed': '435.4', 'train/estimated_remaining_minutes': '6.239'}
+{'loss': '5.431', 'grad_norm': '3.078', 'learning_rate': '0.005', 'epoch': '0.02427', 'train/total_time_seconds': '207.6', 'train/time_per_step_avg': '0.581', 'train/epoch_time_elapsed': '442', 'train/estimated_remaining_minutes': '6.15'}
+{'loss': '5.507', 'grad_norm': '3.141', 'learning_rate': '0.005', 'epoch': '0.02562', 'train/total_time_seconds': '219', 'train/time_per_step_avg': '0.5801', 'train/epoch_time_elapsed': '454.4', 'train/estimated_remaining_minutes': '5.956'}
+{'loss': '5.358', 'grad_norm': '0.2305', 'learning_rate': '0.005', 'epoch': '0.02696', 'train/total_time_seconds': '230.5', 'train/time_per_step_avg': '0.5756', 'train/epoch_time_elapsed': '466.8', 'train/estimated_remaining_minutes': '5.762'}
+{'eval_loss': '5.298', 'eval_runtime': '30.96', 'eval_samples_per_second': '307.7', 'eval_steps_per_second': '2.422', 'epoch': '0.02696', 'train/total_time_seconds': '230.5', 'train/time_per_step_avg': '0.5756', 'train/epoch_time_elapsed': '497.8', 'train/estimated_remaining_minutes': '5.762'}
+{'loss': '5.232', 'grad_norm': '0.6523', 'learning_rate': '0.005', 'epoch': '0.02831', 'train/total_time_seconds': '242', 'train/time_per_step_avg': '0.5773', 'train/epoch_time_elapsed': '510.3', 'train/estimated_remaining_minutes': '5.57'}
+{'loss': '5.145', 'grad_norm': '0.8438', 'learning_rate': '0.005', 'epoch': '0.02966', 'train/total_time_seconds': '253.4', 'train/time_per_step_avg': '0.5772', 'train/epoch_time_elapsed': '522.7', 'train/estimated_remaining_minutes': '5.375'}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-internal.log

@@ -37,3 +37,43 @@
 {"time":"2026-08-12T18:41:54.878519291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T18:42:09.744721347Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
 {"time":"2026-08-12T18:42:09.875728592Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:24.744602672Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:42:24.868798852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:39.745034736Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":32,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:42:39.869583753Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:54.744285656Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":34,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:42:54.886269459Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:09.744477379Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:09.86771103Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:24.744421488Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:43:24.871927529Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:39.744646997Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:39.878977596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:54.744837541Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":2,"events_offset":42,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:54.882795232Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:09.744342836Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:09.912821964Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:24.744203058Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:44:24.907302719Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:39.744333758Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:39.87291761Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:54.744897608Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:54.848473596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:09.744574726Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:09.884635723Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:24.744978196Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:45:24.851010824Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:39.744731176Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:39.88527328Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:54.744980008Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:54.865759988Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:09.744921757Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:09.867433125Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:24.744636764Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:24.850637671Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:39.744161029Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":64,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:46:39.880993457Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:54.744190431Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":2,"events_offset":66,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:54.906911094Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:47:09.744070444Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":68,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:47:09.93423156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/run-qtstmmvv.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e209658e976e1a840bbb10045619606328ca7c92d788a1c688d57a776c374778
-size 196608
+oid sha256:fee906cebe89ea77306f622c76ed457bd79f4dbade0ca525a6ca247575e0a9d2
+size 458752

zain/Activation/wandb/run-20260812_184213-sjvvelzu/files/output.log

@@ -0,0 +1,32 @@
+ 45%|████▌     | 450/1000 [04:41<01:35,  5.77it/s]  
+{'loss': '8.218', 'grad_norm': '1.68', 'learning_rate': '9.5e-05', 'epoch': '0.000337', 'train/total_time_seconds': '3.209', 'train/time_per_step_avg': '0.1605', 'train/epoch_time_elapsed': '3.549', 'train/estimated_remaining_minutes': '2.621'}
+{'loss': '7.848', 'grad_norm': '1.281', 'learning_rate': '0.000195', 'epoch': '0.0006741', 'train/total_time_seconds': '6.413', 'train/time_per_step_avg': '0.1603', 'train/epoch_time_elapsed': '7.036', 'train/estimated_remaining_minutes': '2.565'}
+ 54%|█████▍    | 161/298 [00:14<00:11, 11.48it/s]
+{'eval_loss': '7.331', 'eval_runtime': '25.34', 'eval_samples_per_second': '375.9', 'eval_steps_per_second': '11.76', 'epoch': '0.0008426', 'train/total_time_seconds': '8.007', 'train/time_per_step_avg': '0.1601', 'train/epoch_time_elapsed': '34.12', 'train/estimated_remaining_minutes': '2.535'}
+{'loss': '7.345', 'grad_norm': '1.227', 'learning_rate': '0.000295', 'epoch': '0.001011', 'train/total_time_seconds': '9.623', 'train/time_per_step_avg': '0.1604', 'train/epoch_time_elapsed': '35.88', 'train/estimated_remaining_minutes': '2.513'}
+{'loss': '6.737', 'grad_norm': '1.047', 'learning_rate': '0.000395', 'epoch': '0.001348', 'train/total_time_seconds': '12.84', 'train/time_per_step_avg': '0.1605', 'train/epoch_time_elapsed': '39.38', 'train/estimated_remaining_minutes': '2.461'}
+{'loss': '6.248', 'grad_norm': '0.6719', 'learning_rate': '0.000495', 'epoch': '0.001685', 'train/total_time_seconds': '16.05', 'train/time_per_step_avg': '0.1605', 'train/epoch_time_elapsed': '42.88', 'train/estimated_remaining_minutes': '2.408'}
+{'eval_loss': '6.078', 'eval_runtime': '25.05', 'eval_samples_per_second': '380.3', 'eval_steps_per_second': '11.89', 'epoch': '0.001685', 'train/total_time_seconds': '16.05', 'train/time_per_step_avg': '0.1605', 'train/epoch_time_elapsed': '67.94', 'train/estimated_remaining_minutes': '2.408'}
+{'loss': '5.985', 'grad_norm': '1.812', 'learning_rate': '0.000595', 'epoch': '0.002022', 'train/total_time_seconds': '19.2', 'train/time_per_step_avg': '0.1599', 'train/epoch_time_elapsed': '71.35', 'train/estimated_remaining_minutes': '2.346'}
+{'loss': '5.784', 'grad_norm': '0.5234', 'learning_rate': '0.000695', 'epoch': '0.002359', 'train/total_time_seconds': '22.37', 'train/time_per_step_avg': '0.1596', 'train/epoch_time_elapsed': '74.79', 'train/estimated_remaining_minutes': '2.29'}
+{'eval_loss': '5.561', 'eval_runtime': '25.9', 'eval_samples_per_second': '367.8', 'eval_steps_per_second': '11.51', 'epoch': '0.002528', 'train/total_time_seconds': '23.96', 'train/time_per_step_avg': '0.1595', 'train/epoch_time_elapsed': '102.4', 'train/estimated_remaining_minutes': '2.263'}
+{'loss': '5.571', 'grad_norm': '0.832', 'learning_rate': '0.000795', 'epoch': '0.002696', 'train/total_time_seconds': '25.55', 'train/time_per_step_avg': '0.1593', 'train/epoch_time_elapsed': '104.2', 'train/estimated_remaining_minutes': '2.236'}
+{'loss': '5.355', 'grad_norm': '1.602', 'learning_rate': '0.000895', 'epoch': '0.003033', 'train/total_time_seconds': '28.74', 'train/time_per_step_avg': '0.159', 'train/epoch_time_elapsed': '107.6', 'train/estimated_remaining_minutes': '2.182'}
+{'loss': '5.141', 'grad_norm': '0.5352', 'learning_rate': '0.000995', 'epoch': '0.00337', 'train/total_time_seconds': '32.07', 'train/time_per_step_avg': '0.1602', 'train/epoch_time_elapsed': '111.2', 'train/estimated_remaining_minutes': '2.138'}
+{'eval_loss': '4.981', 'eval_runtime': '24.76', 'eval_samples_per_second': '384.8', 'eval_steps_per_second': '12.04', 'epoch': '0.00337', 'train/total_time_seconds': '32.07', 'train/time_per_step_avg': '0.1602', 'train/epoch_time_elapsed': '136', 'train/estimated_remaining_minutes': '2.138'}
+{'loss': '4.9', 'grad_norm': '0.6055', 'learning_rate': '0.001', 'epoch': '0.003707', 'train/total_time_seconds': '35.18', 'train/time_per_step_avg': '0.1598', 'train/epoch_time_elapsed': '139.4', 'train/estimated_remaining_minutes': '2.079'}
+{'loss': '4.674', 'grad_norm': '0.5898', 'learning_rate': '0.001', 'epoch': '0.004044', 'train/total_time_seconds': '38.38', 'train/time_per_step_avg': '0.1601', 'train/epoch_time_elapsed': '142.9', 'train/estimated_remaining_minutes': '2.026'}
+{'eval_loss': '4.473', 'eval_runtime': '25.77', 'eval_samples_per_second': '369.7', 'eval_steps_per_second': '11.56', 'epoch': '0.004213', 'train/total_time_seconds': '39.97', 'train/time_per_step_avg': '0.1601', 'train/epoch_time_elapsed': '170.4', 'train/estimated_remaining_minutes': '1.998'}
+{'loss': '4.466', 'grad_norm': '0.7852', 'learning_rate': '0.001', 'epoch': '0.004382', 'train/total_time_seconds': '41.6', 'train/time_per_step_avg': '0.1605', 'train/epoch_time_elapsed': '172.1', 'train/estimated_remaining_minutes': '1.973'}
+{'loss': '4.3', 'grad_norm': '0.7617', 'learning_rate': '0.001', 'epoch': '0.004719', 'train/total_time_seconds': '44.8', 'train/time_per_step_avg': '0.1606', 'train/epoch_time_elapsed': '175.6', 'train/estimated_remaining_minutes': '1.92'}
+{'loss': '4.178', 'grad_norm': '0.7305', 'learning_rate': '0.001', 'epoch': '0.005056', 'train/total_time_seconds': '48', 'train/time_per_step_avg': '0.1593', 'train/epoch_time_elapsed': '179.1', 'train/estimated_remaining_minutes': '1.866'}
+{'eval_loss': '4.133', 'eval_runtime': '24.86', 'eval_samples_per_second': '383.2', 'eval_steps_per_second': '11.99', 'epoch': '0.005056', 'train/total_time_seconds': '48', 'train/time_per_step_avg': '0.1593', 'train/epoch_time_elapsed': '204', 'train/estimated_remaining_minutes': '1.866'}
+{'loss': '4.078', 'grad_norm': '0.6367', 'learning_rate': '0.001', 'epoch': '0.005393', 'train/total_time_seconds': '51.27', 'train/time_per_step_avg': '0.1609', 'train/epoch_time_elapsed': '207.5', 'train/estimated_remaining_minutes': '1.816'}
+{'loss': '3.99', 'grad_norm': '0.459', 'learning_rate': '0.001', 'epoch': '0.00573', 'train/total_time_seconds': '54.48', 'train/time_per_step_avg': '0.161', 'train/epoch_time_elapsed': '211', 'train/estimated_remaining_minutes': '1.763'}
+{'eval_loss': '3.91', 'eval_runtime': '25.77', 'eval_samples_per_second': '369.7', 'eval_steps_per_second': '11.56', 'epoch': '0.005898', 'train/total_time_seconds': '56.07', 'train/time_per_step_avg': '0.161', 'train/epoch_time_elapsed': '238.5', 'train/estimated_remaining_minutes': '1.735'}
+{'loss': '3.927', 'grad_norm': '0.7266', 'learning_rate': '0.001', 'epoch': '0.006067', 'train/total_time_seconds': '57.63', 'train/time_per_step_avg': '0.1604', 'train/epoch_time_elapsed': '240.2', 'train/estimated_remaining_minutes': '1.708'}
+{'loss': '3.832', 'grad_norm': '0.4492', 'learning_rate': '0.001', 'epoch': '0.006404', 'train/total_time_seconds': '60.79', 'train/time_per_step_avg': '0.1599', 'train/epoch_time_elapsed': '243.6', 'train/estimated_remaining_minutes': '1.653'}
+{'loss': '3.773', 'grad_norm': '0.5039', 'learning_rate': '0.001', 'epoch': '0.006741', 'train/total_time_seconds': '63.94', 'train/time_per_step_avg': '0.1594', 'train/epoch_time_elapsed': '247.1', 'train/estimated_remaining_minutes': '1.598'}
+{'eval_loss': '3.751', 'eval_runtime': '25.08', 'eval_samples_per_second': '379.9', 'eval_steps_per_second': '11.88', 'epoch': '0.006741', 'train/total_time_seconds': '63.94', 'train/time_per_step_avg': '0.1594', 'train/epoch_time_elapsed': '272.1', 'train/estimated_remaining_minutes': '1.598'}
+{'loss': '3.732', 'grad_norm': '0.459', 'learning_rate': '0.001', 'epoch': '0.007078', 'train/total_time_seconds': '67.23', 'train/time_per_step_avg': '0.1596', 'train/epoch_time_elapsed': '275.7', 'train/estimated_remaining_minutes': '1.547'}
+{'loss': '3.679', 'grad_norm': '0.625', 'learning_rate': '0.001', 'epoch': '0.007415', 'train/total_time_seconds': '70.47', 'train/time_per_step_avg': '0.1599', 'train/epoch_time_elapsed': '279.2', 'train/estimated_remaining_minutes': '1.495'}

zain/Activation/wandb/run-20260812_184213-sjvvelzu/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260812_184213-sjvvelzu/files/wandb-metadata.json

@@ -0,0 +1,107 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-12T18:42:13.957043Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline75l-halfdata.yaml",
+    "--variants",
+    "glu-silu",
+    "glu-relu",
+    "glu-gelu",
+    "glu-sigmoid",
+    "glu-linear",
+    "glu-waleed10",
+    "mlp-waleed10",
+    "glu-silu-waleed10",
+    "mlp-silu-waleed10",
+    "glu-situglu",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "c53713bcba08d1849136dad91288c4b82040634e"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "716183674880"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "d1lsgea9deyyzae97zpdtmme8e3oa7iv"
+}

zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-08-12T18:30:20.635410683Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpuucxrse7/port-1728794.txt","pid":1728794,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-12T18:30:20.635875234Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1728794}
+{"time":"2026-08-12T18:30:20.635869299Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1728794-1730394-4138674846/socket","Net":"unix"}}
+{"time":"2026-08-12T18:30:20.812892363Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-12T18:30:21.015604633Z","level":"INFO","msg":"handleInformInit: received","streamId":"5nis3j8m","id":"1(@)"}
+{"time":"2026-08-12T18:30:21.278746041Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"5nis3j8m","id":"1(@)"}
+{"time":"2026-08-12T18:30:26.79263712Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6en7qisn6akl"}
+{"time":"2026-08-12T18:42:12.551682704Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6en7qisn6akl"}
+{"time":"2026-08-12T18:42:13.174674174Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6en7qisn6akl"}
+{"time":"2026-08-12T18:42:13.176324289Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"5nis3j8m","id":"1(@)"}
+{"time":"2026-08-12T18:42:13.177216893Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"5nis3j8m","id":"1(@)"}
+{"time":"2026-08-12T18:42:13.959420776Z","level":"INFO","msg":"handleInformInit: received","streamId":"sjvvelzu","id":"1(@)"}
+{"time":"2026-08-12T18:42:14.219066208Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"sjvvelzu","id":"1(@)"}
+{"time":"2026-08-12T18:42:19.571080912Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8e1ojla26cxt"}

zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug-internal.log

@@ -0,0 +1,47 @@
+{"time":"2026-08-12T18:42:13.959580652Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T18:42:13.959724138Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T18:42:14.218870288Z","level":"INFO","msg":"stream: created new stream","id":"sjvvelzu"}
+{"time":"2026-08-12T18:42:14.218956754Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T18:42:14.219053701Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T18:42:14.219069228Z","level":"INFO","msg":"writer: started","stream_id":"sjvvelzu"}
+{"time":"2026-08-12T18:42:14.21909666Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T18:42:14.876923676Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:42:14.98880652Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:29.877415684Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":4,"uploaded_len":2}
+{"time":"2026-08-12T18:42:30.005224107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:44.877960291Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:42:44.996668849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:59.877785962Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":4,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:00.000525353Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:14.877602252Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":3,"console_lines":5}
+{"time":"2026-08-12T18:43:15.034494032Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:29.877792727Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":3,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:29.992983921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:44.877553323Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:45.010375305Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:43:59.877295703Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":2,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:43:59.994845826Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:14.877645033Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":2,"events_offset":14,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:15.032619695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:29.877901466Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:44:30.016442098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:44.877582443Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":3,"events_offset":18,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:44:44.989590389Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:44:59.877434616Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:45:00.021745309Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:14.877672035Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":4,"events_offset":22,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:14.996134045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:29.877315928Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:45:30.002914153Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:44.877777926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":26,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:45.003620078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:45:59.877083809Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:45:59.998353063Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:14.877804457Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":2,"events_offset":30,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:15.035374034Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:29.87719442Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":2,"events_offset":32,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:46:29.987619741Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:44.877017477Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:46:45.008164821Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:46:59.87728567Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":3,"events_offset":36,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:47:00.024895002Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug.log
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_184213-sjvvelzu/logs/debug-internal.log
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():772] calling init triggers
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():820] starting backend
+2026-08-12 18:42:13,958 INFO    MainThread:1728794 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 18:42:14,219 INFO    MainThread:1728794 [wandb_init.py:init():840] backend started and connected
+2026-08-12 18:42:14,220 INFO    MainThread:1728794 [wandb_init.py:init():910] updated telemetry
+2026-08-12 18:42:14,227 INFO    MainThread:1728794 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 18:42:14,491 INFO    MainThread:1728794 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 18:42:14,567 INFO    MainThread:1728794 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 18:42:14,568 INFO    MainThread:1728794 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 18:42:14,568 INFO    MainThread:1728794 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 18:42:14,568 INFO    MainThread:1728794 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 18:42:14,569 INFO    MainThread:1728794 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 18:42:14,570 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 75, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'relu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-relu-75L_run', 'per_device_train_batch_size': 16, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-relu-75L-12.8M-20260812-184213', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-relu-75L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 18:42:14,573 INFO    MainThread:1728794 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 12831616 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1523a1b02e90>>
+2026-08-12 18:42:14,573 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 12831616 None

zain/Activation/wandb/run-20260812_184213-sjvvelzu/run-sjvvelzu.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3426bacee0bf502d314a55d67413825d1fc819e2598df3959404689ce570f3a
+size 458752