Auto upload zain 2026-08-12T18:42:09.978601

.gitattributes

@@ -37,3 +37,4 @@ zain/Activation/wandb/run-20260812_175304-hrggki9k/run-hrggki9k.wandb filter=lfs
 zain/Activation/wandb/run-20260812_180713-utqzlg67/run-utqzlg67.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260812_182126-vyycsg6r/run-vyycsg6r.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260812_183021-5nis3j8m/run-5nis3j8m.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260812_183823-qtstmmvv/run-qtstmmvv.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cb9ddfec2a08b0da57eabca99ef5b50c96532af9c99cf2607cc9e73a5ff0f50
+size 50427120

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:844448016e4a6d42d241919d6f90b431ef792ddec5de61eadf3a42feeafca485
+size 101708778

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+size 14244

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db76f238494a413266398860b344c6c4c0f3a045ef78b23af1afac6775a12d1
+size 1064

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,544 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.06740815638692282,
+  "eval_steps": 50,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.000475,
+      "loss": 7.965690612792969,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.0009750000000000001,
+      "loss": 6.832366943359375,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.035778522491455,
+      "eval_runtime": 21.2294,
+      "eval_samples_per_second": 448.765,
+      "eval_steps_per_second": 3.533,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.23046875,
+      "learning_rate": 0.001475,
+      "loss": 6.077784729003906,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.001975,
+      "loss": 6.007767105102539,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 3.78125,
+      "learning_rate": 0.002475,
+      "loss": 5.893904495239258,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 5.818841934204102,
+      "eval_runtime": 21.2871,
+      "eval_samples_per_second": 447.547,
+      "eval_steps_per_second": 3.523,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.21484375,
+      "learning_rate": 0.0029749999999999998,
+      "loss": 5.812347793579102,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 1.328125,
+      "learning_rate": 0.003475,
+      "loss": 5.77868881225586,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 5.751909255981445,
+      "eval_runtime": 21.106,
+      "eval_samples_per_second": 451.388,
+      "eval_steps_per_second": 3.553,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.003975,
+      "loss": 5.767771530151367,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.259765625,
+      "learning_rate": 0.004475,
+      "loss": 5.690555191040039,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.004975,
+      "loss": 5.4286445617675785,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.292172431945801,
+      "eval_runtime": 21.0955,
+      "eval_samples_per_second": 451.613,
+      "eval_steps_per_second": 3.555,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.71484375,
+      "learning_rate": 0.005,
+      "loss": 5.208973693847656,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.005,
+      "loss": 4.965463638305664,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.716352939605713,
+      "eval_runtime": 21.2746,
+      "eval_samples_per_second": 447.81,
+      "eval_steps_per_second": 3.525,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.005,
+      "loss": 4.783096694946289,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.2890625,
+      "learning_rate": 0.005,
+      "loss": 4.766684722900391,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.005,
+      "loss": 4.447724914550781,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.345505237579346,
+      "eval_runtime": 21.1688,
+      "eval_samples_per_second": 450.05,
+      "eval_steps_per_second": 3.543,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.6875,
+      "learning_rate": 0.005,
+      "loss": 4.317182159423828,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.46484375,
+      "learning_rate": 0.005,
+      "loss": 4.167484283447266,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.070701599121094,
+      "eval_runtime": 21.1164,
+      "eval_samples_per_second": 451.165,
+      "eval_steps_per_second": 3.552,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.1015625,
+      "learning_rate": 0.005,
+      "loss": 4.073704528808594,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.005,
+      "loss": 4.146673202514648,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.005,
+      "loss": 3.9575008392333983,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 3.9193575382232666,
+      "eval_runtime": 21.239,
+      "eval_samples_per_second": 448.561,
+      "eval_steps_per_second": 3.531,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.005,
+      "loss": 3.8655147552490234,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.005,
+      "loss": 3.845725250244141,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.8073232173919678,
+      "eval_runtime": 21.1458,
+      "eval_samples_per_second": 450.538,
+      "eval_steps_per_second": 3.547,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.46875,
+      "learning_rate": 0.005,
+      "loss": 3.789488983154297,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.8359375,
+      "learning_rate": 0.005,
+      "loss": 3.744857406616211,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.439453125,
+      "learning_rate": 0.005,
+      "loss": 3.7237689971923826,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.7121422290802,
+      "eval_runtime": 21.3319,
+      "eval_samples_per_second": 446.609,
+      "eval_steps_per_second": 3.516,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.46484375,
+      "learning_rate": 0.005,
+      "loss": 3.6908584594726563,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.51171875,
+      "learning_rate": 0.005,
+      "loss": 3.6516796112060548,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.6792869567871094,
+      "eval_runtime": 21.4305,
+      "eval_samples_per_second": 444.554,
+      "eval_steps_per_second": 3.5,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.005,
+      "loss": 3.6597999572753905,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.005,
+      "loss": 3.6196727752685547,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.005,
+      "loss": 3.606311798095703,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.601555824279785,
+      "eval_runtime": 21.1379,
+      "eval_samples_per_second": 450.706,
+      "eval_steps_per_second": 3.548,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.005,
+      "loss": 3.5888256072998046,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.65234375,
+      "learning_rate": 0.005,
+      "loss": 3.6784610748291016,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.696892261505127,
+      "eval_runtime": 24.9181,
+      "eval_samples_per_second": 382.332,
+      "eval_steps_per_second": 3.01,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.005,
+      "loss": 3.6861244201660157,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.005,
+      "loss": 3.67224006652832,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.005,
+      "loss": 3.6260303497314452,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.642216682434082,
+      "eval_runtime": 30.7072,
+      "eval_samples_per_second": 310.253,
+      "eval_steps_per_second": 2.442,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.796875,
+      "learning_rate": 0.005,
+      "loss": 3.6375244140625,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.005,
+      "loss": 3.6284416198730467,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.597376585006714,
+      "eval_runtime": 30.9199,
+      "eval_samples_per_second": 308.118,
+      "eval_steps_per_second": 2.426,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.005,
+      "loss": 3.597463607788086,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.78125,
+      "learning_rate": 0.005,
+      "loss": 3.6305011749267577,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.6015625,
+      "learning_rate": 0.005,
+      "loss": 3.645455551147461,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 3.589728593826294,
+      "eval_runtime": 31.0369,
+      "eval_samples_per_second": 306.958,
+      "eval_steps_per_second": 2.416,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.5546875,
+      "learning_rate": 0.005,
+      "loss": 3.5748023986816406,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.5,
+      "learning_rate": 0.005,
+      "loss": 3.540536117553711,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 3.5621628761291504,
+      "eval_runtime": 31.1095,
+      "eval_samples_per_second": 306.24,
+      "eval_steps_per_second": 2.411,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.83203125,
+      "learning_rate": 0.005,
+      "loss": 3.5415725708007812,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.78125,
+      "learning_rate": 0.005,
+      "loss": 3.545503616333008,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.005,
+      "loss": 3.532883071899414,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 3.514272451400757,
+      "eval_runtime": 31.133,
+      "eval_samples_per_second": 306.01,
+      "eval_steps_per_second": 2.409,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 9.125,
+      "learning_rate": 0.005,
+      "loss": 3.684223175048828,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 5.34375,
+      "learning_rate": 0.005,
+      "loss": 4.054092788696289,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 4.493913650512695,
+      "eval_runtime": 31.0702,
+      "eval_samples_per_second": 306.629,
+      "eval_steps_per_second": 2.414,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 1.0859375,
+      "learning_rate": 0.005,
+      "loss": 4.461002349853516,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.005,
+      "loss": 4.187630844116211,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.75,
+      "learning_rate": 0.005,
+      "loss": 4.092753601074219,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 3.9804961681365967,
+      "eval_runtime": 31.1964,
+      "eval_samples_per_second": 305.388,
+      "eval_steps_per_second": 2.404,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4839413121024000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-gelu-150L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa39cac60f845af64fe3c6537d152751fa0bf1d92ac306e7cf060b6b8b137f07
+size 4920

zain/Activation/out/glu-gelu-150L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-gelu-150L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cb9ddfec2a08b0da57eabca99ef5b50c96532af9c99cf2607cc9e73a5ff0f50
+size 50427120

zain/Activation/out/glu-gelu-150L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-gelu-150L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa39cac60f845af64fe3c6537d152751fa0bf1d92ac306e7cf060b6b8b137f07
+size 4920

zain/Activation/out/glu-gelu-150L_run/training_log.jsonl

@@ -66,3 +66,7 @@
 {"step": 950, "epoch": 0.06403774856757667, "timestamp": 1786559807.315876, "eval_loss": 4.493913650512695, "eval_runtime": 31.0702, "eval_samples_per_second": 306.629, "eval_steps_per_second": 2.414, "train/total_time_seconds": 407.51738454028964, "train/time_per_step_avg": 0.5732592087239027, "train/epoch_time_elapsed": 920.3366598747671, "train/estimated_remaining_minutes": 0.35747138994762245}
 {"step": 960, "epoch": 0.06471183013144591, "timestamp": 1786559813.6157346, "loss": 4.461002349853516, "grad_norm": 1.0859375, "learning_rate": 0.005, "train/total_time_seconds": 413.29724045470357, "train/time_per_step_avg": 0.5748331969231367, "train/epoch_time_elapsed": 926.6365194283426, "train/estimated_remaining_minutes": 0.2870119725379886}
 {"step": 980, "epoch": 0.06605999325918437, "timestamp": 1786559825.6379988, "loss": 4.187630844116211, "grad_norm": 1.1328125, "learning_rate": 0.005, "train/total_time_seconds": 424.3224572315812, "train/time_per_step_avg": 0.5710843629017472, "train/epoch_time_elapsed": 938.658783391118, "train/estimated_remaining_minutes": 0.14432736640529972}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786559837.95845, "loss": 4.092753601074219, "grad_norm": 0.75, "learning_rate": 0.005, "train/total_time_seconds": 435.6453069075942, "train/time_per_step_avg": 0.5685570389404893, "train/epoch_time_elapsed": 950.979232929647, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786559869.1565416, "eval_loss": 3.9804961681365967, "eval_runtime": 31.1964, "eval_samples_per_second": 305.388, "eval_steps_per_second": 2.404, "train/total_time_seconds": 435.6453069075942, "train/time_per_step_avg": 0.5685570389404893, "train/epoch_time_elapsed": 982.1773252189159, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786559869.5941377, "train_runtime": 983.349, "train_samples_per_second": 65.084, "train_steps_per_second": 1.017, "total_flos": 4839413121024000.0, "train_loss": 4.357875137329102, "train/total_time_seconds": 435.6453069075942, "train/time_per_step_avg": 0.5685570389404893, "train/epoch_time_elapsed": 982.6149224117398, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786559901.1510868, "eval_loss": 3.9804961681365967, "eval_runtime": 31.5534, "eval_samples_per_second": 301.932, "eval_steps_per_second": 2.377, "train/total_time_seconds": 435.6453069075942, "train/time_per_step_avg": 0.5685570389404893, "train/epoch_time_elapsed": 1014.1718704663217, "train/estimated_remaining_minutes": 0.0}

zain/Activation/out/glu-sigmoid-150L_run/training_log.jsonl

@@ -0,0 +1,13 @@
+{"step": 20, "epoch": 0.0013481631277384564, "timestamp": 1786559916.8358912, "loss": 7.948485565185547, "grad_norm": 1.25, "learning_rate": 0.000475, "train/total_time_seconds": 11.955598007887602, "train/time_per_step_avg": 0.5977799003943801, "train/epoch_time_elapsed": 13.052639871835709, "train/estimated_remaining_minutes": 9.763738373108207}
+{"step": 40, "epoch": 0.002696326255476913, "timestamp": 1786559929.1170928, "loss": 6.823790740966797, "grad_norm": 0.84765625, "learning_rate": 0.0009750000000000001, "train/total_time_seconds": 23.237999133765697, "train/time_per_step_avg": 0.5809499783441424, "train/epoch_time_elapsed": 25.333841741085052, "train/estimated_remaining_minutes": 9.295199653506279}
+{"step": 50, "epoch": 0.003370407819346141, "timestamp": 1786559966.298703, "eval_loss": 6.0511932373046875, "eval_runtime": 31.0642, "eval_samples_per_second": 306.688, "eval_steps_per_second": 2.414, "train/total_time_seconds": 28.859849389642477, "train/time_per_step_avg": 0.5771969877928496, "train/epoch_time_elapsed": 62.51545078679919, "train/estimated_remaining_minutes": 9.138952306720117}
+{"step": 60, "epoch": 0.004044489383215369, "timestamp": 1786559972.545297, "loss": 6.1002685546875, "grad_norm": 0.2041015625, "learning_rate": 0.001475, "train/total_time_seconds": 34.61556274071336, "train/time_per_step_avg": 0.576926045678556, "train/epoch_time_elapsed": 68.76204653829336, "train/estimated_remaining_minutes": 9.038508048964044}
+{"step": 80, "epoch": 0.005392652510953826, "timestamp": 1786559985.2270076, "loss": 6.024272537231445, "grad_norm": 0.0966796875, "learning_rate": 0.001975, "train/total_time_seconds": 46.316886223852634, "train/time_per_step_avg": 0.578961077798158, "train/epoch_time_elapsed": 81.44375698640943, "train/estimated_remaining_minutes": 8.877403192905089}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786559997.494285, "loss": 6.017524719238281, "grad_norm": 0.1044921875, "learning_rate": 0.002475, "train/total_time_seconds": 57.606022868305445, "train/time_per_step_avg": 0.5760602286830544, "train/epoch_time_elapsed": 93.71103435754776, "train/estimated_remaining_minutes": 8.640903430245817}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786560028.7062664, "eval_loss": 6.016170024871826, "eval_runtime": 31.2102, "eval_samples_per_second": 305.253, "eval_steps_per_second": 2.403, "train/total_time_seconds": 57.606022868305445, "train/time_per_step_avg": 0.5760602286830544, "train/epoch_time_elapsed": 124.92301358282566, "train/estimated_remaining_minutes": 8.640903430245817}
+{"step": 120, "epoch": 0.008088978766430738, "timestamp": 1786560041.3364766, "loss": 6.028716659545898, "grad_norm": 0.244140625, "learning_rate": 0.0029749999999999998, "train/total_time_seconds": 69.14848026633263, "train/time_per_step_avg": 0.5719288225844502, "train/epoch_time_elapsed": 137.55322621762753, "train/estimated_remaining_minutes": 8.451480921440655}
+{"step": 140, "epoch": 0.009437141894169195, "timestamp": 1786560054.2438993, "loss": 5.9878486633300785, "grad_norm": 0.19140625, "learning_rate": 0.003475, "train/total_time_seconds": 81.070742033422, "train/time_per_step_avg": 0.578327428996563, "train/epoch_time_elapsed": 150.46064829081297, "train/estimated_remaining_minutes": 8.300099779612252}
+{"step": 150, "epoch": 0.010111223458038422, "timestamp": 1786560091.6177757, "eval_loss": 5.871659755706787, "eval_runtime": 31.2412, "eval_samples_per_second": 304.95, "eval_steps_per_second": 2.401, "train/total_time_seconds": 86.70982607454062, "train/time_per_step_avg": 0.5784997668489814, "train/epoch_time_elapsed": 187.83452527597547, "train/estimated_remaining_minutes": 8.189261351484392}
+{"step": 160, "epoch": 0.010785305021907651, "timestamp": 1786560097.858309, "loss": 5.8792579650878904, "grad_norm": 0.365234375, "learning_rate": 0.003975, "train/total_time_seconds": 92.4144375063479, "train/time_per_step_avg": 0.5779887476563453, "train/epoch_time_elapsed": 194.07505892962217, "train/estimated_remaining_minutes": 8.086263281805442}
+{"step": 180, "epoch": 0.012133468149646108, "timestamp": 1786560110.4396544, "loss": 5.723233413696289, "grad_norm": 0.59765625, "learning_rate": 0.004475, "train/total_time_seconds": 104.01449612900615, "train/time_per_step_avg": 0.5769760990515351, "train/epoch_time_elapsed": 206.65640380606055, "train/estimated_remaining_minutes": 7.897396928313429}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786560122.678806, "loss": 5.667977523803711, "grad_norm": 0.296875, "learning_rate": 0.004975, "train/total_time_seconds": 115.22981420531869, "train/time_per_step_avg": 0.5762379133701324, "train/epoch_time_elapsed": 218.89555491134524, "train/estimated_remaining_minutes": 7.6819876136879115}

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 75,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8779561423a0618884138c20f4e68e63c83dba3cd4b6bbfcb49cff5f5cca91d5
+size 25737568

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aa90417f943bed445f9927bc7d6a878b7963206ee35d021a46a6c8a6d0b730d
+size 51903105

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+size 14244

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caa55e3f57ed08f9a32b207aad2408ead6f5f9323ec1ed27900a7c11e3b8bc31
+size 1064

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,544 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.016852039096730706,
+  "eval_steps": 50,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0003370407819346141,
+      "grad_norm": 1.5859375,
+      "learning_rate": 9.5e-05,
+      "loss": 8.218335723876953,
+      "step": 20
+    },
+    {
+      "epoch": 0.0006740815638692282,
+      "grad_norm": 1.28125,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.8475196838378904,
+      "step": 40
+    },
+    {
+      "epoch": 0.0008426019548365353,
+      "eval_loss": 7.329094409942627,
+      "eval_runtime": 25.9476,
+      "eval_samples_per_second": 367.162,
+      "eval_steps_per_second": 11.485,
+      "step": 50
+    },
+    {
+      "epoch": 0.0010111223458038423,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.342254638671875,
+      "step": 60
+    },
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.03125,
+      "learning_rate": 0.000395,
+      "loss": 6.730751800537109,
+      "step": 80
+    },
+    {
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 0.58984375,
+      "learning_rate": 0.000495,
+      "loss": 6.250309753417969,
+      "step": 100
+    },
+    {
+      "epoch": 0.0016852039096730705,
+      "eval_loss": 6.099175453186035,
+      "eval_runtime": 25.0918,
+      "eval_samples_per_second": 379.686,
+      "eval_steps_per_second": 11.876,
+      "step": 100
+    },
+    {
+      "epoch": 0.0020222446916076846,
+      "grad_norm": 0.453125,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 6.0180610656738285,
+      "step": 120
+    },
+    {
+      "epoch": 0.0023592854735422987,
+      "grad_norm": 0.490234375,
+      "learning_rate": 0.000695,
+      "loss": 5.858693695068359,
+      "step": 140
+    },
+    {
+      "epoch": 0.0025278058645096056,
+      "eval_loss": 5.639073371887207,
+      "eval_runtime": 26.2684,
+      "eval_samples_per_second": 362.679,
+      "eval_steps_per_second": 11.344,
+      "step": 150
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.000795,
+      "loss": 5.645732879638672,
+      "step": 160
+    },
+    {
+      "epoch": 0.003033367037411527,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 5.466756439208984,
+      "step": 180
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "grad_norm": 0.671875,
+      "learning_rate": 0.000995,
+      "loss": 5.3429115295410154,
+      "step": 200
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 5.208441257476807,
+      "eval_runtime": 25.4242,
+      "eval_samples_per_second": 374.722,
+      "eval_steps_per_second": 11.721,
+      "step": 200
+    },
+    {
+      "epoch": 0.003707448601280755,
+      "grad_norm": 0.5546875,
+      "learning_rate": 0.001,
+      "loss": 5.11140251159668,
+      "step": 220
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.001,
+      "loss": 4.935234451293946,
+      "step": 240
+    },
+    {
+      "epoch": 0.0042130097741826765,
+      "eval_loss": 4.833581447601318,
+      "eval_runtime": 25.8914,
+      "eval_samples_per_second": 367.96,
+      "eval_steps_per_second": 11.51,
+      "step": 250
+    },
+    {
+      "epoch": 0.004381530165149983,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 4.770340347290039,
+      "step": 260
+    },
+    {
+      "epoch": 0.0047185709470845974,
+      "grad_norm": 2.875,
+      "learning_rate": 0.001,
+      "loss": 4.580703353881836,
+      "step": 280
+    },
+    {
+      "epoch": 0.005055611729019211,
+      "grad_norm": 0.5703125,
+      "learning_rate": 0.001,
+      "loss": 4.454123306274414,
+      "step": 300
+    },
+    {
+      "epoch": 0.005055611729019211,
+      "eval_loss": 4.373719692230225,
+      "eval_runtime": 25.3895,
+      "eval_samples_per_second": 375.235,
+      "eval_steps_per_second": 11.737,
+      "step": 300
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 4.315606689453125,
+      "step": 320
+    },
+    {
+      "epoch": 0.005729693292888439,
+      "grad_norm": 0.6015625,
+      "learning_rate": 0.001,
+      "loss": 4.198415374755859,
+      "step": 340
+    },
+    {
+      "epoch": 0.005898213683855747,
+      "eval_loss": 4.095145225524902,
+      "eval_runtime": 25.5647,
+      "eval_samples_per_second": 372.662,
+      "eval_steps_per_second": 11.657,
+      "step": 350
+    },
+    {
+      "epoch": 0.006066734074823054,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.001,
+      "loss": 4.119716644287109,
+      "step": 360
+    },
+    {
+      "epoch": 0.0064037748567576675,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 4.009439468383789,
+      "step": 380
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.7890625,
+      "learning_rate": 0.001,
+      "loss": 3.9626407623291016,
+      "step": 400
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 3.93593692779541,
+      "eval_runtime": 25.5417,
+      "eval_samples_per_second": 372.998,
+      "eval_steps_per_second": 11.667,
+      "step": 400
+    },
+    {
+      "epoch": 0.007077856420626896,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.001,
+      "loss": 3.9164939880371095,
+      "step": 420
+    },
+    {
+      "epoch": 0.00741489720256151,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.001,
+      "loss": 3.8566329956054686,
+      "step": 440
+    },
+    {
+      "epoch": 0.007583417593528817,
+      "eval_loss": 3.7852697372436523,
+      "eval_runtime": 25.3109,
+      "eval_samples_per_second": 376.399,
+      "eval_steps_per_second": 11.774,
+      "step": 450
+    },
+    {
+      "epoch": 0.007751937984496124,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 3.7881641387939453,
+      "step": 460
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.720836639404297,
+      "step": 480
+    },
+    {
+      "epoch": 0.008426019548365353,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.001,
+      "loss": 3.7119606018066404,
+      "step": 500
+    },
+    {
+      "epoch": 0.008426019548365353,
+      "eval_loss": 3.692281723022461,
+      "eval_runtime": 26.1098,
+      "eval_samples_per_second": 364.882,
+      "eval_steps_per_second": 11.413,
+      "step": 500
+    },
+    {
+      "epoch": 0.008763060330299966,
+      "grad_norm": 0.546875,
+      "learning_rate": 0.001,
+      "loss": 3.656419372558594,
+      "step": 520
+    },
+    {
+      "epoch": 0.00910010111223458,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 3.609619903564453,
+      "step": 540
+    },
+    {
+      "epoch": 0.009268621503201888,
+      "eval_loss": 3.577148675918579,
+      "eval_runtime": 24.7945,
+      "eval_samples_per_second": 384.239,
+      "eval_steps_per_second": 12.019,
+      "step": 550
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.490234375,
+      "learning_rate": 0.001,
+      "loss": 3.5942901611328124,
+      "step": 560
+    },
+    {
+      "epoch": 0.00977418267610381,
+      "grad_norm": 0.58984375,
+      "learning_rate": 0.001,
+      "loss": 3.558675003051758,
+      "step": 580
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "grad_norm": 0.5546875,
+      "learning_rate": 0.001,
+      "loss": 3.486787796020508,
+      "step": 600
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 3.4912281036376953,
+      "eval_runtime": 25.8762,
+      "eval_samples_per_second": 368.176,
+      "eval_steps_per_second": 11.516,
+      "step": 600
+    },
+    {
+      "epoch": 0.010448264239973037,
+      "grad_norm": 0.5390625,
+      "learning_rate": 0.001,
+      "loss": 3.4863636016845705,
+      "step": 620
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.001,
+      "loss": 3.4270252227783202,
+      "step": 640
+    },
+    {
+      "epoch": 0.010953825412874959,
+      "eval_loss": 3.4238741397857666,
+      "eval_runtime": 24.6314,
+      "eval_samples_per_second": 386.782,
+      "eval_steps_per_second": 12.098,
+      "step": 650
+    },
+    {
+      "epoch": 0.011122345803842264,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.001,
+      "loss": 3.4261146545410157,
+      "step": 660
+    },
+    {
+      "epoch": 0.011459386585776879,
+      "grad_norm": 0.56640625,
+      "learning_rate": 0.001,
+      "loss": 3.4203868865966798,
+      "step": 680
+    },
+    {
+      "epoch": 0.011796427367711493,
+      "grad_norm": 0.4765625,
+      "learning_rate": 0.001,
+      "loss": 3.3386905670166014,
+      "step": 700
+    },
+    {
+      "epoch": 0.011796427367711493,
+      "eval_loss": 3.354038715362549,
+      "eval_runtime": 24.3576,
+      "eval_samples_per_second": 391.13,
+      "eval_steps_per_second": 12.234,
+      "step": 700
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.001,
+      "loss": 3.3365108489990236,
+      "step": 720
+    },
+    {
+      "epoch": 0.01247050893158072,
+      "grad_norm": 0.490234375,
+      "learning_rate": 0.001,
+      "loss": 3.3059623718261717,
+      "step": 740
+    },
+    {
+      "epoch": 0.012639029322548028,
+      "eval_loss": 3.28828501701355,
+      "eval_runtime": 25.7521,
+      "eval_samples_per_second": 369.951,
+      "eval_steps_per_second": 11.572,
+      "step": 750
+    },
+    {
+      "epoch": 0.012807549713515335,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.001,
+      "loss": 3.2801753997802736,
+      "step": 760
+    },
+    {
+      "epoch": 0.01314459049544995,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.001,
+      "loss": 3.2589492797851562,
+      "step": 780
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.001,
+      "loss": 3.246501159667969,
+      "step": 800
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.2366974353790283,
+      "eval_runtime": 25.239,
+      "eval_samples_per_second": 377.472,
+      "eval_steps_per_second": 11.807,
+      "step": 800
+    },
+    {
+      "epoch": 0.013818672059319177,
+      "grad_norm": 0.5625,
+      "learning_rate": 0.001,
+      "loss": 3.2192203521728517,
+      "step": 820
+    },
+    {
+      "epoch": 0.014155712841253791,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.001,
+      "loss": 3.1586233139038087,
+      "step": 840
+    },
+    {
+      "epoch": 0.014324233232221099,
+      "eval_loss": 3.175405979156494,
+      "eval_runtime": 25.6686,
+      "eval_samples_per_second": 371.153,
+      "eval_steps_per_second": 11.609,
+      "step": 850
+    },
+    {
+      "epoch": 0.014492753623188406,
+      "grad_norm": 0.486328125,
+      "learning_rate": 0.001,
+      "loss": 3.178068733215332,
+      "step": 860
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.001,
+      "loss": 3.1544485092163086,
+      "step": 880
+    },
+    {
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.447265625,
+      "learning_rate": 0.001,
+      "loss": 3.151500129699707,
+      "step": 900
+    },
+    {
+      "epoch": 0.015166835187057633,
+      "eval_loss": 3.1320953369140625,
+      "eval_runtime": 25.0538,
+      "eval_samples_per_second": 380.262,
+      "eval_steps_per_second": 11.894,
+      "step": 900
+    },
+    {
+      "epoch": 0.015503875968992248,
+      "grad_norm": 0.470703125,
+      "learning_rate": 0.001,
+      "loss": 3.113129806518555,
+      "step": 920
+    },
+    {
+      "epoch": 0.015840916750926862,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 3.079757308959961,
+      "step": 940
+    },
+    {
+      "epoch": 0.016009437141894168,
+      "eval_loss": 3.085155963897705,
+      "eval_runtime": 25.2701,
+      "eval_samples_per_second": 377.007,
+      "eval_steps_per_second": 11.793,
+      "step": 950
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.462890625,
+      "learning_rate": 0.001,
+      "loss": 3.0605375289916994,
+      "step": 960
+    },
+    {
+      "epoch": 0.01651499831479609,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.001,
+      "loss": 3.0471120834350587,
+      "step": 980
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.001,
+      "loss": 3.05130615234375,
+      "step": 1000
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 3.045870542526245,
+      "eval_runtime": 25.3615,
+      "eval_samples_per_second": 375.648,
+      "eval_steps_per_second": 11.75,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 604929785856000.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-75L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:954b9896ded9312046b3a63252dc6a327ab94f4db2083972bb5b01198e71fda8
+size 4920

zain/Activation/out/glu-silu-75L_run/training_log.jsonl

@@ -39,3 +39,33 @@
 {"step": 560, "epoch": 0.009437141894169195, "timestamp": 1786559803.6180458, "loss": 3.5942901611328124, "grad_norm": 0.490234375, "learning_rate": 0.001, "train/total_time_seconds": 92.52027648314834, "train/time_per_step_avg": 0.16243518318980932, "train/epoch_time_elapsed": 381.8204213306308, "train/estimated_remaining_minutes": 1.2115750491840853}
 {"step": 580, "epoch": 0.00977418267610381, "timestamp": 1786559807.1333785, "loss": 3.558675003051758, "grad_norm": 0.58984375, "learning_rate": 0.001, "train/total_time_seconds": 95.7451943717897, "train/time_per_step_avg": 0.16219746198505164, "train/epoch_time_elapsed": 385.3357537537813, "train/estimated_remaining_minutes": 1.1555454493147033}
 {"step": 600, "epoch": 0.010111223458038422, "timestamp": 1786559810.6723864, "loss": 3.486787796020508, "grad_norm": 0.5546875, "learning_rate": 0.001, "train/total_time_seconds": 99.0130730420351, "train/time_per_step_avg": 0.16271060578525065, "train/epoch_time_elapsed": 388.8747618943453, "train/estimated_remaining_minutes": 1.1001452560226122}
+{"step": 600, "epoch": 0.010111223458038422, "timestamp": 1786559836.5501053, "eval_loss": 3.4912281036376953, "eval_runtime": 25.8762, "eval_samples_per_second": 368.176, "eval_steps_per_second": 11.516, "train/total_time_seconds": 99.0130730420351, "train/time_per_step_avg": 0.16271060578525065, "train/epoch_time_elapsed": 414.7524805292487, "train/estimated_remaining_minutes": 1.1001452560226122}
+{"step": 620, "epoch": 0.010448264239973037, "timestamp": 1786559840.0896788, "loss": 3.4863636016845705, "grad_norm": 0.5390625, "learning_rate": 0.001, "train/total_time_seconds": 102.26573623344302, "train/time_per_step_avg": 0.16209108874201775, "train/epoch_time_elapsed": 418.2920548580587, "train/estimated_remaining_minutes": 1.0446499937824825}
+{"step": 640, "epoch": 0.010785305021907651, "timestamp": 1786559843.5125225, "loss": 3.4270252227783202, "grad_norm": 0.5234375, "learning_rate": 0.001, "train/total_time_seconds": 105.41327896341681, "train/time_per_step_avg": 0.16080061085522174, "train/epoch_time_elapsed": 421.7148975133896, "train/estimated_remaining_minutes": 0.9882494902820326}
+{"step": 650, "epoch": 0.010953825412874959, "timestamp": 1786559869.9014323, "eval_loss": 3.4238741397857666, "eval_runtime": 24.6314, "eval_samples_per_second": 386.782, "eval_steps_per_second": 12.098, "train/total_time_seconds": 107.03115490078926, "train/time_per_step_avg": 0.16086759097874165, "train/epoch_time_elapsed": 448.1038069911301, "train/estimated_remaining_minutes": 0.9605360055199036}
+{"step": 660, "epoch": 0.011122345803842264, "timestamp": 1786559871.5795712, "loss": 3.4261146545410157, "grad_norm": 0.51953125, "learning_rate": 0.001, "train/total_time_seconds": 108.56900959461927, "train/time_per_step_avg": 0.16048733111470936, "train/epoch_time_elapsed": 449.78194618970156, "train/estimated_remaining_minutes": 0.9321581631861251}
+{"step": 680, "epoch": 0.011459386585776879, "timestamp": 1786559875.000632, "loss": 3.4203868865966798, "grad_norm": 0.56640625, "learning_rate": 0.001, "train/total_time_seconds": 111.71837348490953, "train/time_per_step_avg": 0.1597317911311984, "train/epoch_time_elapsed": 453.2030081227422, "train/estimated_remaining_minutes": 0.8762225371365453}
+{"step": 700, "epoch": 0.011796427367711493, "timestamp": 1786559878.3987033, "loss": 3.3386905670166014, "grad_norm": 0.4765625, "learning_rate": 0.001, "train/total_time_seconds": 114.8433718830347, "train/time_per_step_avg": 0.15830298840999604, "train/epoch_time_elapsed": 456.60107857361436, "train/estimated_remaining_minutes": 0.8203097991645336}
+{"step": 700, "epoch": 0.011796427367711493, "timestamp": 1786559902.7579427, "eval_loss": 3.354038715362549, "eval_runtime": 24.3576, "eval_samples_per_second": 391.13, "eval_steps_per_second": 12.234, "train/total_time_seconds": 114.8433718830347, "train/time_per_step_avg": 0.15830298840999604, "train/epoch_time_elapsed": 480.9603180848062, "train/estimated_remaining_minutes": 0.8203097991645336}
+{"step": 720, "epoch": 0.012133468149646108, "timestamp": 1786559906.0499735, "loss": 3.3365108489990236, "grad_norm": 0.451171875, "learning_rate": 0.001, "train/total_time_seconds": 117.86583146080375, "train/time_per_step_avg": 0.15600095227360725, "train/epoch_time_elapsed": 484.2523493953049, "train/estimated_remaining_minutes": 0.7639452039126168}
+{"step": 740, "epoch": 0.01247050893158072, "timestamp": 1786559909.4807403, "loss": 3.3059623718261717, "grad_norm": 0.490234375, "learning_rate": 0.001, "train/total_time_seconds": 121.02717741206288, "train/time_per_step_avg": 0.15613898448646069, "train/epoch_time_elapsed": 487.6831152327359, "train/estimated_remaining_minutes": 0.708717705566134}
+{"step": 750, "epoch": 0.012639029322548028, "timestamp": 1786559937.0405338, "eval_loss": 3.28828501701355, "eval_runtime": 25.7521, "eval_samples_per_second": 369.951, "eval_steps_per_second": 11.572, "train/total_time_seconds": 122.69024388864636, "train/time_per_step_avg": 0.15659088987857103, "train/epoch_time_elapsed": 515.2429086752236, "train/estimated_remaining_minutes": 0.6816124660480353}
+{"step": 760, "epoch": 0.012807549713515335, "timestamp": 1786559938.7460802, "loss": 3.2801753997802736, "grad_norm": 0.427734375, "learning_rate": 0.001, "train/total_time_seconds": 124.25834885239601, "train/time_per_step_avg": 0.15689339257776738, "train/epoch_time_elapsed": 516.94845501706, "train/estimated_remaining_minutes": 0.6539913097494527}
+{"step": 780, "epoch": 0.01314459049544995, "timestamp": 1786559942.2240026, "loss": 3.2589492797851562, "grad_norm": 0.53515625, "learning_rate": 0.001, "train/total_time_seconds": 127.4633274152875, "train/time_per_step_avg": 0.1574495393037796, "train/epoch_time_elapsed": 520.4263781532645, "train/estimated_remaining_minutes": 0.5991865818667361}
+{"step": 800, "epoch": 0.013481631277384564, "timestamp": 1786559945.7809155, "loss": 3.246501159667969, "grad_norm": 0.48046875, "learning_rate": 0.001, "train/total_time_seconds": 130.72742968797684, "train/time_per_step_avg": 0.1588405780494213, "train/epoch_time_elapsed": 523.9832903929055, "train/estimated_remaining_minutes": 0.5446976236999035}
+{"step": 800, "epoch": 0.013481631277384564, "timestamp": 1786559971.0216343, "eval_loss": 3.2366974353790283, "eval_runtime": 25.239, "eval_samples_per_second": 377.472, "eval_steps_per_second": 11.807, "train/total_time_seconds": 130.72742968797684, "train/time_per_step_avg": 0.1588405780494213, "train/epoch_time_elapsed": 549.2240088097751, "train/estimated_remaining_minutes": 0.5446976236999035}
+{"step": 820, "epoch": 0.013818672059319177, "timestamp": 1786559974.516833, "loss": 3.2192203521728517, "grad_norm": 0.5625, "learning_rate": 0.001, "train/total_time_seconds": 133.9457576572895, "train/time_per_step_avg": 0.16079926196485758, "train/epoch_time_elapsed": 552.7192084379494, "train/estimated_remaining_minutes": 0.4900454548437421}
+{"step": 840, "epoch": 0.014155712841253791, "timestamp": 1786559978.0194778, "loss": 3.1586233139038087, "grad_norm": 0.458984375, "learning_rate": 0.001, "train/total_time_seconds": 137.17295909672976, "train/time_per_step_avg": 0.1614578168466687, "train/epoch_time_elapsed": 556.221852786839, "train/estimated_remaining_minutes": 0.4354697114181897}
+{"step": 850, "epoch": 0.014324233232221099, "timestamp": 1786560005.4207242, "eval_loss": 3.175405979156494, "eval_runtime": 25.6686, "eval_samples_per_second": 371.153, "eval_steps_per_second": 11.609, "train/total_time_seconds": 138.76898622140288, "train/time_per_step_avg": 0.16078742332756518, "train/epoch_time_elapsed": 583.6230989769101, "train/estimated_remaining_minutes": 0.4081440771217732}
+{"step": 860, "epoch": 0.014492753623188406, "timestamp": 1786560007.1150353, "loss": 3.178068733215332, "grad_norm": 0.486328125, "learning_rate": 0.001, "train/total_time_seconds": 140.32343230769038, "train/time_per_step_avg": 0.1606508345529437, "train/epoch_time_elapsed": 585.3174117915332, "train/estimated_remaining_minutes": 0.38072249075729947}
+{"step": 880, "epoch": 0.01482979440512302, "timestamp": 1786560010.493896, "loss": 3.1544485092163086, "grad_norm": 0.515625, "learning_rate": 0.001, "train/total_time_seconds": 143.42693192884326, "train/time_per_step_avg": 0.15963604513555765, "train/epoch_time_elapsed": 588.6962721794844, "train/estimated_remaining_minutes": 0.32597029983828013}
+{"step": 900, "epoch": 0.015166835187057633, "timestamp": 1786560013.9152408, "loss": 3.151500129699707, "grad_norm": 0.447265625, "learning_rate": 0.001, "train/total_time_seconds": 146.5769969969988, "train/time_per_step_avg": 0.1584956730902195, "train/epoch_time_elapsed": 592.1176167055964, "train/estimated_remaining_minutes": 0.2714388833277755}
+{"step": 900, "epoch": 0.015166835187057633, "timestamp": 1786560038.9705796, "eval_loss": 3.1320953369140625, "eval_runtime": 25.0538, "eval_samples_per_second": 380.262, "eval_steps_per_second": 11.894, "train/total_time_seconds": 146.5769969969988, "train/time_per_step_avg": 0.1584956730902195, "train/epoch_time_elapsed": 617.1729544401169, "train/estimated_remaining_minutes": 0.2714388833277755}
+{"step": 920, "epoch": 0.015503875968992248, "timestamp": 1786560042.36617, "loss": 3.113129806518555, "grad_norm": 0.470703125, "learning_rate": 0.001, "train/total_time_seconds": 149.7022238485515, "train/time_per_step_avg": 0.15756466191262006, "train/epoch_time_elapsed": 620.5685461610556, "train/estimated_remaining_minutes": 0.2169597447080457}
+{"step": 940, "epoch": 0.015840916750926862, "timestamp": 1786560045.836104, "loss": 3.079757308959961, "grad_norm": 0.50390625, "learning_rate": 0.001, "train/total_time_seconds": 152.9016332104802, "train/time_per_step_avg": 0.15728674113750457, "train/epoch_time_elapsed": 624.0384798571467, "train/estimated_remaining_minutes": 0.16266131192604277}
+{"step": 950, "epoch": 0.016009437141894168, "timestamp": 1786560072.784192, "eval_loss": 3.085155963897705, "eval_runtime": 25.2701, "eval_samples_per_second": 377.007, "eval_steps_per_second": 11.793, "train/total_time_seconds": 154.443891197443, "train/time_per_step_avg": 0.15674904976040124, "train/epoch_time_elapsed": 650.9865671060979, "train/estimated_remaining_minutes": 0.1354770975416167}
+{"step": 960, "epoch": 0.016177957532861477, "timestamp": 1786560074.4459326, "loss": 3.0605375289916994, "grad_norm": 0.462890625, "learning_rate": 0.001, "train/total_time_seconds": 155.97115107625723, "train/time_per_step_avg": 0.15647718768566846, "train/epoch_time_elapsed": 652.6483084112406, "train/estimated_remaining_minutes": 0.10831329935851196}
+{"step": 980, "epoch": 0.01651499831479609, "timestamp": 1786560077.8585532, "loss": 3.0471120834350587, "grad_norm": 0.4375, "learning_rate": 0.001, "train/total_time_seconds": 159.10734697058797, "train/time_per_step_avg": 0.15680415041744708, "train/epoch_time_elapsed": 656.060926835984, "train/estimated_remaining_minutes": 0.05411814522809114}
+{"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560081.5580728, "loss": 3.05130615234375, "grad_norm": 0.59765625, "learning_rate": 0.001, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 659.760448589921, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560106.9211233, "eval_loss": 3.045870542526245, "eval_runtime": 25.3615, "eval_samples_per_second": 375.648, "eval_steps_per_second": 11.75, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 685.1234988421202, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.016852039096730706, "timestamp": 1786560107.1479647, "train_runtime": 686.5633, "train_samples_per_second": 23.304, "train_steps_per_second": 1.457, "total_flos": 604929785856000.0, "train_loss": 4.216384292602539, "train/total_time_seconds": 162.52781854197383, "train/time_per_step_avg": 0.1595082154497504, "train/epoch_time_elapsed": 685.3503403961658, "train/estimated_remaining_minutes": 0.0}

zain/Activation/wandb/debug-internal.log

@@ -1,63 +1,39 @@
-{"time":"2026-08-12T18:30:21.015765765Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-12T18:30:21.016057414Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-12T18:30:21.278570088Z","level":"INFO","msg":"stream: created new stream","id":"5nis3j8m"}
-{"time":"2026-08-12T18:30:21.278652455Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-12T18:30:21.278738829Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-12T18:30:21.278762324Z","level":"INFO","msg":"writer: started","stream_id":"5nis3j8m"}
-{"time":"2026-08-12T18:30:21.278772779Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-12T18:30:21.910888229Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T18:30:22.010925758Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:30:36.911710852Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":5,"uploaded_len":2}
-{"time":"2026-08-12T18:30:37.037504996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:30:51.911357707Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1,"events_lines":2,"console_offset":5,"console_lines":1}
-{"time":"2026-08-12T18:30:52.076200771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:31:06.911256577Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":4,"events_offset":3,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:31:07.052259904Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:31:21.911232721Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":5,"console_lines":5}
-{"time":"2026-08-12T18:31:22.031654868Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:31:36.911182866Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":2,"events_offset":7,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:31:37.024946801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:31:51.911133053Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":9,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:31:52.020721611Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:32:06.911431987Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":11,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:32:07.056371976Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:32:21.911345244Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":3,"events_offset":13,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:32:22.022749251Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:32:36.911545207Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":15,"events_lines":2,"console_offset":5,"console_lines":1}
-{"time":"2026-08-12T18:32:37.020821587Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:32:51.911271581Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":3,"events_offset":17,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:32:52.056047823Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:33:06.910953115Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":5,"console_lines":1}
-{"time":"2026-08-12T18:33:07.046059342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:33:21.911181442Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":3,"events_offset":21,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:33:22.062156357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:33:36.911677043Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":23,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:33:37.021006925Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:33:51.911163322Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":25,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:33:52.022909336Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:34:06.911856217Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":2,"events_offset":27,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:34:07.067457311Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:34:21.912502507Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":29,"events_lines":2,"console_offset":5,"console_lines":1}
-{"time":"2026-08-12T18:34:22.067202274Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:34:36.911119503Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":4,"events_offset":31,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:34:37.026433606Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:34:51.910977316Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":33,"events_lines":2,"console_offset":5,"console_lines":1}
-{"time":"2026-08-12T18:34:52.020397547Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:35:06.911350101Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":3,"events_offset":35,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:35:07.024983957Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:35:21.911231031Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":37,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:35:22.034404837Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:35:36.911879943Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":2,"events_offset":39,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-12T18:35:37.049120953Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:35:51.911850759Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":2,"events_offset":41,"events_lines":2,"console_offset":5,"console_lines":1}
-{"time":"2026-08-12T18:35:52.015760508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:36:06.911612719Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":43,"events_lines":2,"console_offset":10,"console_lines":28}
-{"time":"2026-08-12T18:36:07.048574088Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:36:21.911547421Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":3,"events_offset":45,"events_lines":2,"console_offset":38,"console_lines":3}
-{"time":"2026-08-12T18:36:22.069688877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:36:36.912036526Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":47,"events_lines":2}
-{"time":"2026-08-12T18:36:37.033432478Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:36:51.911128246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":4,"events_offset":49,"events_lines":2,"console_offset":41,"console_lines":4}
-{"time":"2026-08-12T18:36:52.013606109Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T18:37:06.91125731Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":51,"events_lines":2}
-{"time":"2026-08-12T18:37:07.061438591Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:23.105010622Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T18:38:23.105150417Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T18:38:23.371916432Z","level":"INFO","msg":"stream: created new stream","id":"qtstmmvv"}
+{"time":"2026-08-12T18:38:23.372014762Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T18:38:23.372116804Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T18:38:23.372128482Z","level":"INFO","msg":"writer: started","stream_id":"qtstmmvv"}
+{"time":"2026-08-12T18:38:23.372151282Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T18:38:24.743864927Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:38:24.838824501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:39.744346348Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-12T18:38:39.869953996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:54.744630635Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:38:54.899949077Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:09.744851347Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:39:09.862852929Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:24.744295359Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":2,"console_lines":2}
+{"time":"2026-08-12T18:39:24.889674817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:39.744771093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:39:39.90389517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:54.744830696Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:39:54.85572055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:09.74451592Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:40:09.918048601Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:24.744095296Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":3,"console_lines":5}
+{"time":"2026-08-12T18:40:24.871071918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:39.744522438Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:40:39.878594588Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:54.744332246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":18,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:40:54.86474185Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:09.744339141Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:41:09.862772827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:24.744324362Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:41:24.869002616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:39.744710255Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":2,"events_offset":24,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:41:39.854076154Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:54.744214155Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:41:54.878519291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:09.744721347Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:42:09.875728592Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/debug.log

@@ -1,22 +1,20 @@
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_setup.py:_flush():81] Configure stats pid to 1728794
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183021-5nis3j8m/logs/debug.log
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183021-5nis3j8m/logs/debug-internal.log
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_init.py:init():772] calling init triggers
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug.log
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-internal.log
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_init.py:init():820] starting backend
-2026-08-12 18:30:21,014 INFO    MainThread:1728794 [wandb_init.py:init():835] sending inform_init request
-2026-08-12 18:30:21,279 INFO    MainThread:1728794 [wandb_init.py:init():840] backend started and connected
-2026-08-12 18:30:21,280 INFO    MainThread:1728794 [wandb_init.py:init():910] updated telemetry
-2026-08-12 18:30:21,287 INFO    MainThread:1728794 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-12 18:30:21,708 INFO    MainThread:1728794 [wandb_init.py:init():978] starting run threads in backend
-2026-08-12 18:30:21,789 INFO    MainThread:1728794 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-12 18:30:21,789 INFO    MainThread:1728794 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-12 18:30:21,789 INFO    MainThread:1728794 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-12 18:30:21,790 INFO    MainThread:1728794 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-12 18:30:21,793 INFO    MainThread:1728794 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-12 18:30:21,793 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 75, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-75L_run', 'per_device_train_batch_size': 16, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-75L-12.8M-20260812-183019', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-silu-75L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-12 18:30:21,797 INFO    MainThread:1728794 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 12831616 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1523cf1900d0>>
-2026-08-12 18:30:21,797 INFO    MainThread:1728794 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 12831616 None
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 18:38:23,372 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
+2026-08-12 18:38:23,374 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
+2026-08-12 18:38:23,380 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 18:38:23,694 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 18:38:23,774 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 18:38:23,774 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 18:38:23,776 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'sigmoid', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-sigmoid-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-sigmoid-150L-25.1M-20260812-183822', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-sigmoid-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 18:38:23,782 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c5940d9050>>
+2026-08-12 18:38:23,783 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None

zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug-core.log

@@ -32,3 +32,10 @@
 {"time":"2026-08-12T18:21:26.321522195Z","level":"INFO","msg":"handleInformInit: received","streamId":"vyycsg6r","id":"3(@)"}
 {"time":"2026-08-12T18:21:26.578729863Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vyycsg6r","id":"3(@)"}
 {"time":"2026-08-12T18:21:31.999599169Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.284226521Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.818456712Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.81999609Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:21.828229465Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.104854149Z","level":"INFO","msg":"handleInformInit: received","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.372128976Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:28.777282159Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}

zain/Activation/wandb/run-20260812_180713-utqzlg67/logs/debug-core.log

@@ -32,3 +32,10 @@
 {"time":"2026-08-12T18:21:26.321522195Z","level":"INFO","msg":"handleInformInit: received","streamId":"vyycsg6r","id":"3(@)"}
 {"time":"2026-08-12T18:21:26.578729863Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vyycsg6r","id":"3(@)"}
 {"time":"2026-08-12T18:21:31.999599169Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.284226521Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.818456712Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.81999609Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:21.828229465Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.104854149Z","level":"INFO","msg":"handleInformInit: received","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.372128976Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:28.777282159Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}

zain/Activation/wandb/run-20260812_182126-vyycsg6r/files/config.yaml

@@ -0,0 +1,445 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            sgrmmsib7zyio51wnd7qcvhe5i7gdczy:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml
+                    - --variants
+                    - glu-silu
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-linear
+                    - glu-waleed10
+                    - mlp-waleed10
+                    - glu-silu-waleed10
+                    - mlp-silu-waleed10
+                    - glu-situglu
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "716178280448"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: c53713bcba08d1849136dad91288c4b82040634e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-12T18:21:26.319176Z"
+                writerId: sgrmmsib7zyio51wnd7qcvhe5i7gdczy
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: gelu
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 50
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-gelu-150L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.005
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 0
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 25138816
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 150
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-gelu-150L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 128
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-gelu-150L-25.1M-20260812-182125
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260812_182126-vyycsg6r/files/output.log

@@ -68,3 +68,16 @@
 {'eval_loss': '4.494', 'eval_runtime': '31.07', 'eval_samples_per_second': '306.6', 'eval_steps_per_second': '2.414', 'epoch': '0.06404', 'train/total_time_seconds': '407.5', 'train/time_per_step_avg': '0.5733', 'train/epoch_time_elapsed': '920.3', 'train/estimated_remaining_minutes': '0.3575'}
 {'loss': '4.461', 'grad_norm': '1.086', 'learning_rate': '0.005', 'epoch': '0.06471', 'train/total_time_seconds': '413.3', 'train/time_per_step_avg': '0.5748', 'train/epoch_time_elapsed': '926.6', 'train/estimated_remaining_minutes': '0.287'}
 {'loss': '4.188', 'grad_norm': '1.133', 'learning_rate': '0.005', 'epoch': '0.06606', 'train/total_time_seconds': '424.3', 'train/time_per_step_avg': '0.5711', 'train/epoch_time_elapsed': '938.7', 'train/estimated_remaining_minutes': '0.1443'}
+{'loss': '4.093', 'grad_norm': '0.75', 'learning_rate': '0.005', 'epoch': '0.06741', 'train/total_time_seconds': '435.6', 'train/time_per_step_avg': '0.5686', 'train/epoch_time_elapsed': '951', 'train/estimated_remaining_minutes': '0'}
+{'eval_loss': '3.98', 'eval_runtime': '31.2', 'eval_samples_per_second': '305.4', 'eval_steps_per_second': '2.404', 'epoch': '0.06741', 'train/total_time_seconds': '435.6', 'train/time_per_step_avg': '0.5686', 'train/epoch_time_elapsed': '982.2', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00,  9.30it/s]
+100%|██████████| 1000/1000 [16:23<00:00,  1.02it/s]0:00,  9.39it/s]
+{'train_runtime': '983.3', 'train_samples_per_second': '65.08', 'train_steps_per_second': '1.017', 'train_loss': '4.358', 'epoch': '0.06741', 'train/total_time_seconds': '435.6', 'train/time_per_step_avg': '0.5686', 'train/epoch_time_elapsed': '982.6', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 75/75 [00:31<00:00,  2.41it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 10.94it/s]

zain/Activation/wandb/run-20260812_182126-vyycsg6r/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/loss":3.9804961681365967,"train_runtime":983.349,"train/train/epoch_time_elapsed":1014.1718704663217,"_runtime":1014,"total_flos":4.839413121024e+15,"eval/runtime":31.5534,"train/train/total_time_seconds":435.6453069075942,"_wandb":{"runtime":1014},"train_loss":4.357875137329102,"train/epoch":0.06740815638692282,"eval/samples_per_second":301.932,"train/train/estimated_remaining_minutes":0,"train_steps_per_second":1.017,"train/grad_norm":0.75,"_timestamp":1.7865599011513355e+09,"train/train/time_per_step_avg":0.5685570389404893,"train/global_step":1000,"eval/steps_per_second":2.377,"train/loss":4.092753601074219,"_step":71,"train/learning_rate":0.005,"train_samples_per_second":65.084}

zain/Activation/wandb/run-20260812_182126-vyycsg6r/logs/debug-core.log

@@ -32,3 +32,10 @@
 {"time":"2026-08-12T18:21:26.321522195Z","level":"INFO","msg":"handleInformInit: received","streamId":"vyycsg6r","id":"3(@)"}
 {"time":"2026-08-12T18:21:26.578729863Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vyycsg6r","id":"3(@)"}
 {"time":"2026-08-12T18:21:31.999599169Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.284226521Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.818456712Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.81999609Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:21.828229465Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.104854149Z","level":"INFO","msg":"handleInformInit: received","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.372128976Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:28.777282159Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}

zain/Activation/wandb/run-20260812_182126-vyycsg6r/logs/debug-internal.log

@@ -131,3 +131,21 @@
 {"time":"2026-08-12T18:36:43.0449338Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T18:36:57.884412932Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":2,"events_offset":122,"events_lines":2,"console_offset":67,"console_lines":2}
 {"time":"2026-08-12T18:36:57.997968481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:12.884324783Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":67,"history_lines":1,"events_offset":124,"events_lines":2,"console_offset":69,"console_lines":1}
+{"time":"2026-08-12T18:37:12.99508184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:27.884119301Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":68,"history_lines":1,"events_offset":126,"events_lines":2,"console_offset":70,"console_lines":1}
+{"time":"2026-08-12T18:37:28.031354504Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:42.883920799Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":128,"events_lines":2}
+{"time":"2026-08-12T18:37:43.000151349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:57.884397768Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":2,"events_offset":130,"events_lines":2,"console_offset":71,"console_lines":7}
+{"time":"2026-08-12T18:37:57.993176058Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:12.884643682Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":132,"events_lines":2,"console_offset":77,"console_lines":1}
+{"time":"2026-08-12T18:38:13.018992357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:21.663852511Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-12T18:38:21.664020269Z","level":"INFO","msg":"filestream: sending request","total_files":3,"history_offset":71,"history_lines":1,"console_offset":77,"console_lines":6,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-12T18:38:21.816198843Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:21.817244677Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-12T18:38:21.820028193Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-12T18:38:21.820044826Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-12T18:38:21.827824114Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-12T18:38:21.827842903Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260812_182126-vyycsg6r/logs/debug.log

@@ -18,3 +18,8 @@ config: {'_wandb': {}}
 2026-08-12 18:21:26,971 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'gelu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-gelu-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-gelu-150L-25.1M-20260812-182125', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-gelu-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-12 18:21:26,978 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c5efbb9090>>
 2026-08-12 18:21:26,979 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None
+2026-08-12 18:38:21,283 INFO    MainThread:1371664 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research2/vyycsg6r
+2026-08-12 18:38:21,283 INFO    MainThread:1371664 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-12 18:38:21,283 INFO    MainThread:1371664 [wandb_run.py:_restore():2570] restore
+2026-08-12 18:38:21,284 INFO    MainThread:1371664 [wandb_run.py:_restore():2576] restore done
+2026-08-12 18:38:21,819 INFO    MainThread:1371664 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260812_182126-vyycsg6r/run-vyycsg6r.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c51648b8c50008e59eb3b3dd2b030e6acfdcfd82b168b01a4443d51d0d2c5eb2
-size 917504
+oid sha256:e7aad41382f986f15ba3d15eeeacd7fd68546f2b3005a5653aa27fffde18db39
+size 1004128

zain/Activation/wandb/run-20260812_183021-5nis3j8m/files/output.log

@@ -43,3 +43,38 @@
 {'loss': '3.594', 'grad_norm': '0.4902', 'learning_rate': '0.001', 'epoch': '0.009437', 'train/total_time_seconds': '92.52', 'train/time_per_step_avg': '0.1624', 'train/epoch_time_elapsed': '381.8', 'train/estimated_remaining_minutes': '1.212'}
 {'loss': '3.559', 'grad_norm': '0.5898', 'learning_rate': '0.001', 'epoch': '0.009774', 'train/total_time_seconds': '95.75', 'train/time_per_step_avg': '0.1622', 'train/epoch_time_elapsed': '385.3', 'train/estimated_remaining_minutes': '1.156'}
 {'loss': '3.487', 'grad_norm': '0.5547', 'learning_rate': '0.001', 'epoch': '0.01011', 'train/total_time_seconds': '99.01', 'train/time_per_step_avg': '0.1627', 'train/epoch_time_elapsed': '388.9', 'train/estimated_remaining_minutes': '1.1'}
+{'eval_loss': '3.491', 'eval_runtime': '25.88', 'eval_samples_per_second': '368.2', 'eval_steps_per_second': '11.52', 'epoch': '0.01011', 'train/total_time_seconds': '99.01', 'train/time_per_step_avg': '0.1627', 'train/epoch_time_elapsed': '414.8', 'train/estimated_remaining_minutes': '1.1'}
+{'loss': '3.486', 'grad_norm': '0.5391', 'learning_rate': '0.001', 'epoch': '0.01045', 'train/total_time_seconds': '102.3', 'train/time_per_step_avg': '0.1621', 'train/epoch_time_elapsed': '418.3', 'train/estimated_remaining_minutes': '1.045'}
+{'loss': '3.427', 'grad_norm': '0.5234', 'learning_rate': '0.001', 'epoch': '0.01079', 'train/total_time_seconds': '105.4', 'train/time_per_step_avg': '0.1608', 'train/epoch_time_elapsed': '421.7', 'train/estimated_remaining_minutes': '0.9882'}
+{'eval_loss': '3.424', 'eval_runtime': '24.63', 'eval_samples_per_second': '386.8', 'eval_steps_per_second': '12.1', 'epoch': '0.01095', 'train/total_time_seconds': '107', 'train/time_per_step_avg': '0.1609', 'train/epoch_time_elapsed': '448.1', 'train/estimated_remaining_minutes': '0.9605'}
+{'loss': '3.426', 'grad_norm': '0.5195', 'learning_rate': '0.001', 'epoch': '0.01112', 'train/total_time_seconds': '108.6', 'train/time_per_step_avg': '0.1605', 'train/epoch_time_elapsed': '449.8', 'train/estimated_remaining_minutes': '0.9322'}
+{'loss': '3.42', 'grad_norm': '0.5664', 'learning_rate': '0.001', 'epoch': '0.01146', 'train/total_time_seconds': '111.7', 'train/time_per_step_avg': '0.1597', 'train/epoch_time_elapsed': '453.2', 'train/estimated_remaining_minutes': '0.8762'}
+{'loss': '3.339', 'grad_norm': '0.4766', 'learning_rate': '0.001', 'epoch': '0.0118', 'train/total_time_seconds': '114.8', 'train/time_per_step_avg': '0.1583', 'train/epoch_time_elapsed': '456.6', 'train/estimated_remaining_minutes': '0.8203'}
+{'eval_loss': '3.354', 'eval_runtime': '24.36', 'eval_samples_per_second': '391.1', 'eval_steps_per_second': '12.23', 'epoch': '0.0118', 'train/total_time_seconds': '114.8', 'train/time_per_step_avg': '0.1583', 'train/epoch_time_elapsed': '481', 'train/estimated_remaining_minutes': '0.8203'}
+{'loss': '3.337', 'grad_norm': '0.4512', 'learning_rate': '0.001', 'epoch': '0.01213', 'train/total_time_seconds': '117.9', 'train/time_per_step_avg': '0.156', 'train/epoch_time_elapsed': '484.3', 'train/estimated_remaining_minutes': '0.7639'}
+{'loss': '3.306', 'grad_norm': '0.4902', 'learning_rate': '0.001', 'epoch': '0.01247', 'train/total_time_seconds': '121', 'train/time_per_step_avg': '0.1561', 'train/epoch_time_elapsed': '487.7', 'train/estimated_remaining_minutes': '0.7087'}
+{'eval_loss': '3.288', 'eval_runtime': '25.75', 'eval_samples_per_second': '370', 'eval_steps_per_second': '11.57', 'epoch': '0.01264', 'train/total_time_seconds': '122.7', 'train/time_per_step_avg': '0.1566', 'train/epoch_time_elapsed': '515.2', 'train/estimated_remaining_minutes': '0.6816'}
+{'loss': '3.28', 'grad_norm': '0.4277', 'learning_rate': '0.001', 'epoch': '0.01281', 'train/total_time_seconds': '124.3', 'train/time_per_step_avg': '0.1569', 'train/epoch_time_elapsed': '516.9', 'train/estimated_remaining_minutes': '0.654'}
+{'loss': '3.259', 'grad_norm': '0.5352', 'learning_rate': '0.001', 'epoch': '0.01314', 'train/total_time_seconds': '127.5', 'train/time_per_step_avg': '0.1574', 'train/epoch_time_elapsed': '520.4', 'train/estimated_remaining_minutes': '0.5992'}
+{'loss': '3.247', 'grad_norm': '0.4805', 'learning_rate': '0.001', 'epoch': '0.01348', 'train/total_time_seconds': '130.7', 'train/time_per_step_avg': '0.1588', 'train/epoch_time_elapsed': '524', 'train/estimated_remaining_minutes': '0.5447'}
+{'eval_loss': '3.237', 'eval_runtime': '25.24', 'eval_samples_per_second': '377.5', 'eval_steps_per_second': '11.81', 'epoch': '0.01348', 'train/total_time_seconds': '130.7', 'train/time_per_step_avg': '0.1588', 'train/epoch_time_elapsed': '549.2', 'train/estimated_remaining_minutes': '0.5447'}
+{'loss': '3.219', 'grad_norm': '0.5625', 'learning_rate': '0.001', 'epoch': '0.01382', 'train/total_time_seconds': '133.9', 'train/time_per_step_avg': '0.1608', 'train/epoch_time_elapsed': '552.7', 'train/estimated_remaining_minutes': '0.49'}
+{'loss': '3.159', 'grad_norm': '0.459', 'learning_rate': '0.001', 'epoch': '0.01416', 'train/total_time_seconds': '137.2', 'train/time_per_step_avg': '0.1615', 'train/epoch_time_elapsed': '556.2', 'train/estimated_remaining_minutes': '0.4355'}
+{'eval_loss': '3.175', 'eval_runtime': '25.67', 'eval_samples_per_second': '371.2', 'eval_steps_per_second': '11.61', 'epoch': '0.01432', 'train/total_time_seconds': '138.8', 'train/time_per_step_avg': '0.1608', 'train/epoch_time_elapsed': '583.6', 'train/estimated_remaining_minutes': '0.4081'}
+{'loss': '3.178', 'grad_norm': '0.4863', 'learning_rate': '0.001', 'epoch': '0.01449', 'train/total_time_seconds': '140.3', 'train/time_per_step_avg': '0.1607', 'train/epoch_time_elapsed': '585.3', 'train/estimated_remaining_minutes': '0.3807'}
+{'loss': '3.154', 'grad_norm': '0.5156', 'learning_rate': '0.001', 'epoch': '0.01483', 'train/total_time_seconds': '143.4', 'train/time_per_step_avg': '0.1596', 'train/epoch_time_elapsed': '588.7', 'train/estimated_remaining_minutes': '0.326'}
+{'loss': '3.152', 'grad_norm': '0.4473', 'learning_rate': '0.001', 'epoch': '0.01517', 'train/total_time_seconds': '146.6', 'train/time_per_step_avg': '0.1585', 'train/epoch_time_elapsed': '592.1', 'train/estimated_remaining_minutes': '0.2714'}
+{'eval_loss': '3.132', 'eval_runtime': '25.05', 'eval_samples_per_second': '380.3', 'eval_steps_per_second': '11.89', 'epoch': '0.01517', 'train/total_time_seconds': '146.6', 'train/time_per_step_avg': '0.1585', 'train/epoch_time_elapsed': '617.2', 'train/estimated_remaining_minutes': '0.2714'}
+{'loss': '3.113', 'grad_norm': '0.4707', 'learning_rate': '0.001', 'epoch': '0.0155', 'train/total_time_seconds': '149.7', 'train/time_per_step_avg': '0.1576', 'train/epoch_time_elapsed': '620.6', 'train/estimated_remaining_minutes': '0.217'}
+{'loss': '3.08', 'grad_norm': '0.5039', 'learning_rate': '0.001', 'epoch': '0.01584', 'train/total_time_seconds': '152.9', 'train/time_per_step_avg': '0.1573', 'train/epoch_time_elapsed': '624', 'train/estimated_remaining_minutes': '0.1627'}
+{'eval_loss': '3.085', 'eval_runtime': '25.27', 'eval_samples_per_second': '377', 'eval_steps_per_second': '11.79', 'epoch': '0.01601', 'train/total_time_seconds': '154.4', 'train/time_per_step_avg': '0.1567', 'train/epoch_time_elapsed': '651', 'train/estimated_remaining_minutes': '0.1355'}
+{'loss': '3.061', 'grad_norm': '0.4629', 'learning_rate': '0.001', 'epoch': '0.01618', 'train/total_time_seconds': '156', 'train/time_per_step_avg': '0.1565', 'train/epoch_time_elapsed': '652.6', 'train/estimated_remaining_minutes': '0.1083'}
+{'loss': '3.047', 'grad_norm': '0.4375', 'learning_rate': '0.001', 'epoch': '0.01651', 'train/total_time_seconds': '159.1', 'train/time_per_step_avg': '0.1568', 'train/epoch_time_elapsed': '656.1', 'train/estimated_remaining_minutes': '0.05412'}
+{'loss': '3.051', 'grad_norm': '0.5977', 'learning_rate': '0.001', 'epoch': '0.01685', 'train/total_time_seconds': '162.5', 'train/time_per_step_avg': '0.1595', 'train/epoch_time_elapsed': '659.8', 'train/estimated_remaining_minutes': '0'}
+{'eval_loss': '3.046', 'eval_runtime': '25.36', 'eval_samples_per_second': '375.6', 'eval_steps_per_second': '11.75', 'epoch': '0.01685', 'train/total_time_seconds': '162.5', 'train/time_per_step_avg': '0.1595', 'train/epoch_time_elapsed': '685.1', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.20it/s]
+100%|██████████| 1000/1000 [11:26<00:00,  1.46it/s], ?it/s]
+{'train_runtime': '686.6', 'train_samples_per_second': '23.3', 'train_steps_per_second': '1.457', 'train_loss': '4.216', 'epoch': '0.01685', 'train/total_time_seconds': '162.5', 'train/time_per_step_avg': '0.1595', 'train/epoch_time_elapsed': '685.4', 'train/estimated_remaining_minutes': '0'}
+ 78%|███████▊  | 231/298 [00:19<00:05, 11.75it/s]

zain/Activation/wandb/run-20260812_183021-5nis3j8m/logs/debug-internal.log

@@ -61,3 +61,43 @@
 {"time":"2026-08-12T18:36:52.013606109Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T18:37:06.91125731Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":51,"events_lines":2}
 {"time":"2026-08-12T18:37:07.061438591Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:21.91198923Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":2,"events_offset":53,"events_lines":2,"console_offset":45,"console_lines":2}
+{"time":"2026-08-12T18:37:22.027899674Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:36.911448333Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":1,"events_offset":55,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-08-12T18:37:37.018605108Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:37:51.911140787Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":44,"history_lines":2,"events_offset":57,"events_lines":2,"console_offset":48,"console_lines":2}
+{"time":"2026-08-12T18:37:52.023732721Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:06.911218117Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":2,"events_offset":59,"events_lines":2,"console_offset":50,"console_lines":2}
+{"time":"2026-08-12T18:38:07.06003466Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:21.911195348Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":61,"events_lines":2}
+{"time":"2026-08-12T18:38:22.030458784Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:36.911386615Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":3,"events_offset":63,"events_lines":2,"console_offset":52,"console_lines":3}
+{"time":"2026-08-12T18:38:37.029603649Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:51.911351149Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":65,"events_lines":2}
+{"time":"2026-08-12T18:38:52.019780527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:06.911629884Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":51,"history_lines":4,"events_offset":67,"events_lines":2,"console_offset":55,"console_lines":4}
+{"time":"2026-08-12T18:39:07.053250903Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:21.911760758Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":69,"events_lines":2}
+{"time":"2026-08-12T18:39:22.030508392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:36.911859233Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":2,"events_offset":71,"events_lines":2,"console_offset":59,"console_lines":2}
+{"time":"2026-08-12T18:39:37.055018096Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:51.911906735Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":57,"history_lines":1,"events_offset":73,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-12T18:39:52.026783491Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:06.911154685Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":1,"events_offset":75,"events_lines":2,"console_offset":62,"console_lines":1}
+{"time":"2026-08-12T18:40:07.064235027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:21.911453002Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":3,"events_offset":77,"events_lines":2,"console_offset":63,"console_lines":3}
+{"time":"2026-08-12T18:40:22.032889006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:36.911846359Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":79,"events_lines":2}
+{"time":"2026-08-12T18:40:37.022374085Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:51.911847123Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":62,"history_lines":3,"events_offset":81,"events_lines":2,"console_offset":66,"console_lines":3}
+{"time":"2026-08-12T18:40:52.032356671Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:06.911228942Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":83,"events_lines":2}
+{"time":"2026-08-12T18:41:07.040742634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:21.911715772Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":4,"events_offset":85,"events_lines":2,"console_offset":69,"console_lines":4}
+{"time":"2026-08-12T18:41:22.046450737Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:36.911787732Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":87,"events_lines":2}
+{"time":"2026-08-12T18:41:37.023801143Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:51.911199515Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":2,"events_offset":89,"events_lines":2,"console_offset":73,"console_lines":7}
+{"time":"2026-08-12T18:41:52.059907127Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:06.911175834Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":91,"events_lines":2,"console_offset":79,"console_lines":1}
+{"time":"2026-08-12T18:42:07.087160098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_183021-5nis3j8m/run-5nis3j8m.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b338f191513b210d2a5a4f109e4c4fcffaf39932fc42df74945367bc2f79178
-size 655360
+oid sha256:a950a00b950559e2a29e05461ef1e292bef6e43a390c0f6f573164f8f4f9ebde
+size 1114112

zain/Activation/wandb/run-20260812_183823-qtstmmvv/files/output.log

@@ -0,0 +1,15 @@
+ 20%|██        | 200/1000 [03:39<08:01,  1.66it/s]  
+{'loss': '7.948', 'grad_norm': '1.25', 'learning_rate': '0.000475', 'epoch': '0.001348', 'train/total_time_seconds': '11.96', 'train/time_per_step_avg': '0.5978', 'train/epoch_time_elapsed': '13.05', 'train/estimated_remaining_minutes': '9.764'}
+{'loss': '6.824', 'grad_norm': '0.8477', 'learning_rate': '0.000975', 'epoch': '0.002696', 'train/total_time_seconds': '23.24', 'train/time_per_step_avg': '0.5809', 'train/epoch_time_elapsed': '25.33', 'train/estimated_remaining_minutes': '9.295'}
+ 23%|██▎       | 17/75 [00:06<00:24,  2.37it/s]
+{'eval_loss': '6.051', 'eval_runtime': '31.06', 'eval_samples_per_second': '306.7', 'eval_steps_per_second': '2.414', 'epoch': '0.00337', 'train/total_time_seconds': '28.86', 'train/time_per_step_avg': '0.5772', 'train/epoch_time_elapsed': '62.52', 'train/estimated_remaining_minutes': '9.139'}
+{'loss': '6.1', 'grad_norm': '0.2041', 'learning_rate': '0.001475', 'epoch': '0.004044', 'train/total_time_seconds': '34.62', 'train/time_per_step_avg': '0.5769', 'train/epoch_time_elapsed': '68.76', 'train/estimated_remaining_minutes': '9.039'}
+{'loss': '6.024', 'grad_norm': '0.09668', 'learning_rate': '0.001975', 'epoch': '0.005393', 'train/total_time_seconds': '46.32', 'train/time_per_step_avg': '0.579', 'train/epoch_time_elapsed': '81.44', 'train/estimated_remaining_minutes': '8.877'}
+{'loss': '6.018', 'grad_norm': '0.1045', 'learning_rate': '0.002475', 'epoch': '0.006741', 'train/total_time_seconds': '57.61', 'train/time_per_step_avg': '0.5761', 'train/epoch_time_elapsed': '93.71', 'train/estimated_remaining_minutes': '8.641'}
+{'eval_loss': '6.016', 'eval_runtime': '31.21', 'eval_samples_per_second': '305.3', 'eval_steps_per_second': '2.403', 'epoch': '0.006741', 'train/total_time_seconds': '57.61', 'train/time_per_step_avg': '0.5761', 'train/epoch_time_elapsed': '124.9', 'train/estimated_remaining_minutes': '8.641'}
+{'loss': '6.029', 'grad_norm': '0.2441', 'learning_rate': '0.002975', 'epoch': '0.008089', 'train/total_time_seconds': '69.15', 'train/time_per_step_avg': '0.5719', 'train/epoch_time_elapsed': '137.6', 'train/estimated_remaining_minutes': '8.451'}
+{'loss': '5.988', 'grad_norm': '0.1914', 'learning_rate': '0.003475', 'epoch': '0.009437', 'train/total_time_seconds': '81.07', 'train/time_per_step_avg': '0.5783', 'train/epoch_time_elapsed': '150.5', 'train/estimated_remaining_minutes': '8.3'}
+{'eval_loss': '5.872', 'eval_runtime': '31.24', 'eval_samples_per_second': '304.9', 'eval_steps_per_second': '2.401', 'epoch': '0.01011', 'train/total_time_seconds': '86.71', 'train/time_per_step_avg': '0.5785', 'train/epoch_time_elapsed': '187.8', 'train/estimated_remaining_minutes': '8.189'}
+{'loss': '5.879', 'grad_norm': '0.3652', 'learning_rate': '0.003975', 'epoch': '0.01079', 'train/total_time_seconds': '92.41', 'train/time_per_step_avg': '0.578', 'train/epoch_time_elapsed': '194.1', 'train/estimated_remaining_minutes': '8.086'}
+{'loss': '5.723', 'grad_norm': '0.5977', 'learning_rate': '0.004475', 'epoch': '0.01213', 'train/total_time_seconds': '104', 'train/time_per_step_avg': '0.577', 'train/epoch_time_elapsed': '206.7', 'train/estimated_remaining_minutes': '7.897'}
+{'loss': '5.668', 'grad_norm': '0.2969', 'learning_rate': '0.004975', 'epoch': '0.01348', 'train/total_time_seconds': '115.2', 'train/time_per_step_avg': '0.5762', 'train/epoch_time_elapsed': '218.9', 'train/estimated_remaining_minutes': '7.682'}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260812_183823-qtstmmvv/files/wandb-metadata.json

@@ -0,0 +1,107 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-12T18:38:23.093072Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml",
+    "--variants",
+    "glu-silu",
+    "glu-relu",
+    "glu-gelu",
+    "glu-sigmoid",
+    "glu-linear",
+    "glu-waleed10",
+    "mlp-waleed10",
+    "glu-silu-waleed10",
+    "mlp-silu-waleed10",
+    "glu-situglu",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "c53713bcba08d1849136dad91288c4b82040634e"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "716186701824"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "fi9sjbwy7wx2qoke6aps07jyl5wn8xrl"
+}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-core.log

@@ -0,0 +1,41 @@
+{"time":"2026-08-12T17:51:29.496049185Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7lru5pey/port-1355517.txt","pid":1355517,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-12T17:51:29.496747364Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1355517}
+{"time":"2026-08-12T17:51:29.49671311Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1355517-1357782-3052194627/socket","Net":"unix"}}
+{"time":"2026-08-12T17:51:29.674347603Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-12T17:52:11.162138091Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"2(@)"}
+{"time":"2026-08-12T17:52:11.242645021Z","level":"INFO","msg":"handleInformInit: received","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:11.508233203Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:17.001520061Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.412124918Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.887487538Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.888505Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:32.889313366Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597228944Z","level":"INFO","msg":"connection: closing","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597314364Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597242804Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597322441Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-12T17:53:03.98517338Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-12T17:53:04.076747848Z","level":"INFO","msg":"handleInformInit: received","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T17:53:04.339071164Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T17:53:09.739880255Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:11.667315857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:12.178747012Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:12.180598654Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T18:07:12.181733156Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T18:07:13.412475061Z","level":"INFO","msg":"handleInformInit: received","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:07:13.679314069Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:07:19.044675117Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:24.445193258Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:25.196343989Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:25.198067009Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:21:25.199000312Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:21:26.321522195Z","level":"INFO","msg":"handleInformInit: received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:21:26.578729863Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:21:31.999599169Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.284226521Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.818456712Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.81999609Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:21.828229465Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.104854149Z","level":"INFO","msg":"handleInformInit: received","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.372128976Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:28.777282159Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-internal.log

@@ -0,0 +1,39 @@
+{"time":"2026-08-12T18:38:23.105010622Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T18:38:23.105150417Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T18:38:23.371916432Z","level":"INFO","msg":"stream: created new stream","id":"qtstmmvv"}
+{"time":"2026-08-12T18:38:23.372014762Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T18:38:23.372116804Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T18:38:23.372128482Z","level":"INFO","msg":"writer: started","stream_id":"qtstmmvv"}
+{"time":"2026-08-12T18:38:23.372151282Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T18:38:24.743864927Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:38:24.838824501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:39.744346348Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-12T18:38:39.869953996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:38:54.744630635Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:38:54.899949077Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:09.744851347Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:39:09.862852929Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:24.744295359Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":2,"console_lines":2}
+{"time":"2026-08-12T18:39:24.889674817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:39.744771093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:39:39.90389517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:39:54.744830696Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:39:54.85572055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:09.74451592Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:40:09.918048601Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:24.744095296Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":3,"console_lines":5}
+{"time":"2026-08-12T18:40:24.871071918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:39.744522438Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:40:39.878594588Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:40:54.744332246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":18,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:40:54.86474185Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:09.744339141Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:41:09.862772827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:24.744324362Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-12T18:41:24.869002616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:39.744710255Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":2,"events_offset":24,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:41:39.854076154Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:41:54.744214155Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:41:54.878519291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:42:09.744721347Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:42:09.875728592Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug.log
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-internal.log
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-12 18:38:23,094 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 18:38:23,372 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
+2026-08-12 18:38:23,374 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
+2026-08-12 18:38:23,380 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 18:38:23,694 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 18:38:23,774 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 18:38:23,774 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 18:38:23,775 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 18:38:23,776 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'sigmoid', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-sigmoid-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-sigmoid-150L-25.1M-20260812-183822', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-sigmoid-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 18:38:23,782 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c5940d9050>>
+2026-08-12 18:38:23,783 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None

zain/Activation/wandb/run-20260812_183823-qtstmmvv/run-qtstmmvv.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e209658e976e1a840bbb10045619606328ca7c92d788a1c688d57a776c374778
+size 196608