Auto upload zain 2026-08-14T20:28:11.532417 (part 2)

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:405f33cd7fd059ef025bd7455617606d70568c7106b07c1995d2c44d11066298
+size 4011496

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d93d508a39585ae3d0b82f4135d50740ca4c206e0190908db8717fbe6c445e1
+size 8074746

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5502244c7ba8b61ab0a3b01cdc7e5e2629060a0565d6b782445d6ba31022cee
+size 1064

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/trainer_state.json

@@ -0,0 +1,244 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.32345013477088946,
+  "eval_steps": 60000,
+  "global_step": 600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01078167115902965,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.001,
+      "loss": 7.1676788330078125,
+      "step": 20
+    },
+    {
+      "epoch": 0.0215633423180593,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.001,
+      "loss": 5.993187332153321,
+      "step": 40
+    },
+    {
+      "epoch": 0.03234501347708895,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.001,
+      "loss": 5.496040725708008,
+      "step": 60
+    },
+    {
+      "epoch": 0.0431266846361186,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.001,
+      "loss": 4.934689331054687,
+      "step": 80
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.001,
+      "loss": 4.479576110839844,
+      "step": 100
+    },
+    {
+      "epoch": 0.0646900269541779,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.001,
+      "loss": 4.171505355834961,
+      "step": 120
+    },
+    {
+      "epoch": 0.07547169811320754,
+      "grad_norm": 0.55078125,
+      "learning_rate": 0.001,
+      "loss": 3.940719223022461,
+      "step": 140
+    },
+    {
+      "epoch": 0.0862533692722372,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.752634811401367,
+      "step": 160
+    },
+    {
+      "epoch": 0.09703504043126684,
+      "grad_norm": 0.4765625,
+      "learning_rate": 0.001,
+      "loss": 3.6034828186035157,
+      "step": 180
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 3.4787067413330077,
+      "step": 200
+    },
+    {
+      "epoch": 0.11859838274932614,
+      "grad_norm": 0.9296875,
+      "learning_rate": 0.001,
+      "loss": 3.3636749267578123,
+      "step": 220
+    },
+    {
+      "epoch": 0.1293800539083558,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.269498825073242,
+      "step": 240
+    },
+    {
+      "epoch": 0.14016172506738545,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 3.183236503601074,
+      "step": 260
+    },
+    {
+      "epoch": 0.1509433962264151,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.001,
+      "loss": 3.099096488952637,
+      "step": 280
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.8359375,
+      "learning_rate": 0.001,
+      "loss": 3.0244571685791017,
+      "step": 300
+    },
+    {
+      "epoch": 0.1725067385444744,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.001,
+      "loss": 2.9644407272338866,
+      "step": 320
+    },
+    {
+      "epoch": 0.18328840970350405,
+      "grad_norm": 0.83984375,
+      "learning_rate": 0.001,
+      "loss": 2.8989635467529298,
+      "step": 340
+    },
+    {
+      "epoch": 0.1940700808625337,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.001,
+      "loss": 2.851881980895996,
+      "step": 360
+    },
+    {
+      "epoch": 0.20485175202156333,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 2.790276527404785,
+      "step": 380
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.7615821838378904,
+      "step": 400
+    },
+    {
+      "epoch": 0.22641509433962265,
+      "grad_norm": 0.83203125,
+      "learning_rate": 0.001,
+      "loss": 2.7132925033569335,
+      "step": 420
+    },
+    {
+      "epoch": 0.2371967654986523,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.691997528076172,
+      "step": 440
+    },
+    {
+      "epoch": 0.24797843665768193,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.6550079345703126,
+      "step": 460
+    },
+    {
+      "epoch": 0.2587601078167116,
+      "grad_norm": 0.58984375,
+      "learning_rate": 0.001,
+      "loss": 2.6208978652954102,
+      "step": 480
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.001,
+      "loss": 2.593034362792969,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803234501347709,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.001,
+      "loss": 2.5671127319335936,
+      "step": 520
+    },
+    {
+      "epoch": 0.29110512129380056,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 2.5521316528320312,
+      "step": 540
+    },
+    {
+      "epoch": 0.3018867924528302,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.5245645523071287,
+      "step": 560
+    },
+    {
+      "epoch": 0.31266846361185985,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 2.5037845611572265,
+      "step": 580
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.74609375,
+      "learning_rate": 0.001,
+      "loss": 2.482448196411133,
+      "step": 600
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1393864527052800.0,
+  "train_batch_size": 512,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-600/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea57eafa5d8d7c92fa44d42011d21fc8a9b019b0c1ee506d1d2f3d7c5eb1dccd
+size 4920

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c717a54e659c9a2512a557b6b891ba338f420dd308778ffc1a91d63ac53f6fce
+size 4011496

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1693a3c3e3b6ef9e2c17f0dc8ab3df78e06baba9ca316eafe33b55c31cf9e50c
+size 8074746

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3173cd6908322b02fdacbfeb79bbb6a1a1a40d218262cceb79ee8c11c4fa07d7
+size 1064

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/trainer_state.json

@@ -0,0 +1,279 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.37735849056603776,
+  "eval_steps": 60000,
+  "global_step": 700,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01078167115902965,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.001,
+      "loss": 7.1676788330078125,
+      "step": 20
+    },
+    {
+      "epoch": 0.0215633423180593,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.001,
+      "loss": 5.993187332153321,
+      "step": 40
+    },
+    {
+      "epoch": 0.03234501347708895,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.001,
+      "loss": 5.496040725708008,
+      "step": 60
+    },
+    {
+      "epoch": 0.0431266846361186,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.001,
+      "loss": 4.934689331054687,
+      "step": 80
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.001,
+      "loss": 4.479576110839844,
+      "step": 100
+    },
+    {
+      "epoch": 0.0646900269541779,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.001,
+      "loss": 4.171505355834961,
+      "step": 120
+    },
+    {
+      "epoch": 0.07547169811320754,
+      "grad_norm": 0.55078125,
+      "learning_rate": 0.001,
+      "loss": 3.940719223022461,
+      "step": 140
+    },
+    {
+      "epoch": 0.0862533692722372,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.752634811401367,
+      "step": 160
+    },
+    {
+      "epoch": 0.09703504043126684,
+      "grad_norm": 0.4765625,
+      "learning_rate": 0.001,
+      "loss": 3.6034828186035157,
+      "step": 180
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 3.4787067413330077,
+      "step": 200
+    },
+    {
+      "epoch": 0.11859838274932614,
+      "grad_norm": 0.9296875,
+      "learning_rate": 0.001,
+      "loss": 3.3636749267578123,
+      "step": 220
+    },
+    {
+      "epoch": 0.1293800539083558,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.269498825073242,
+      "step": 240
+    },
+    {
+      "epoch": 0.14016172506738545,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 3.183236503601074,
+      "step": 260
+    },
+    {
+      "epoch": 0.1509433962264151,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.001,
+      "loss": 3.099096488952637,
+      "step": 280
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.8359375,
+      "learning_rate": 0.001,
+      "loss": 3.0244571685791017,
+      "step": 300
+    },
+    {
+      "epoch": 0.1725067385444744,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.001,
+      "loss": 2.9644407272338866,
+      "step": 320
+    },
+    {
+      "epoch": 0.18328840970350405,
+      "grad_norm": 0.83984375,
+      "learning_rate": 0.001,
+      "loss": 2.8989635467529298,
+      "step": 340
+    },
+    {
+      "epoch": 0.1940700808625337,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.001,
+      "loss": 2.851881980895996,
+      "step": 360
+    },
+    {
+      "epoch": 0.20485175202156333,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 2.790276527404785,
+      "step": 380
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.7615821838378904,
+      "step": 400
+    },
+    {
+      "epoch": 0.22641509433962265,
+      "grad_norm": 0.83203125,
+      "learning_rate": 0.001,
+      "loss": 2.7132925033569335,
+      "step": 420
+    },
+    {
+      "epoch": 0.2371967654986523,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.691997528076172,
+      "step": 440
+    },
+    {
+      "epoch": 0.24797843665768193,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.6550079345703126,
+      "step": 460
+    },
+    {
+      "epoch": 0.2587601078167116,
+      "grad_norm": 0.58984375,
+      "learning_rate": 0.001,
+      "loss": 2.6208978652954102,
+      "step": 480
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.001,
+      "loss": 2.593034362792969,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803234501347709,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.001,
+      "loss": 2.5671127319335936,
+      "step": 520
+    },
+    {
+      "epoch": 0.29110512129380056,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 2.5521316528320312,
+      "step": 540
+    },
+    {
+      "epoch": 0.3018867924528302,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.5245645523071287,
+      "step": 560
+    },
+    {
+      "epoch": 0.31266846361185985,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 2.5037845611572265,
+      "step": 580
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.74609375,
+      "learning_rate": 0.001,
+      "loss": 2.482448196411133,
+      "step": 600
+    },
+    {
+      "epoch": 0.33423180592991913,
+      "grad_norm": 1.078125,
+      "learning_rate": 0.001,
+      "loss": 2.4656560897827147,
+      "step": 620
+    },
+    {
+      "epoch": 0.3450134770889488,
+      "grad_norm": 0.5703125,
+      "learning_rate": 0.001,
+      "loss": 2.4476362228393556,
+      "step": 640
+    },
+    {
+      "epoch": 0.3557951482479784,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 2.4330963134765624,
+      "step": 660
+    },
+    {
+      "epoch": 0.3665768194070081,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.41370849609375,
+      "step": 680
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.397682952880859,
+      "step": 700
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1626175281561600.0,
+  "train_batch_size": 512,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-700/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea57eafa5d8d7c92fa44d42011d21fc8a9b019b0c1ee506d1d2f3d7c5eb1dccd
+size 4920

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72be666a6d8a687c82848db9da6e226474463313935a2105343cc622f8304407
+size 4011496

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70760fb060951c44c000636dee740b936a2684d721e835fa55ebba9882a1a9c9
+size 8074746

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7334026bbd250befa466dcbe6bec8bfe84c03024263e4dea55e40698f2caa92e
+size 1064

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/trainer_state.json

@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.431266846361186,
+  "eval_steps": 60000,
+  "global_step": 800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01078167115902965,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.001,
+      "loss": 7.1676788330078125,
+      "step": 20
+    },
+    {
+      "epoch": 0.0215633423180593,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.001,
+      "loss": 5.993187332153321,
+      "step": 40
+    },
+    {
+      "epoch": 0.03234501347708895,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.001,
+      "loss": 5.496040725708008,
+      "step": 60
+    },
+    {
+      "epoch": 0.0431266846361186,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.001,
+      "loss": 4.934689331054687,
+      "step": 80
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.001,
+      "loss": 4.479576110839844,
+      "step": 100
+    },
+    {
+      "epoch": 0.0646900269541779,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.001,
+      "loss": 4.171505355834961,
+      "step": 120
+    },
+    {
+      "epoch": 0.07547169811320754,
+      "grad_norm": 0.55078125,
+      "learning_rate": 0.001,
+      "loss": 3.940719223022461,
+      "step": 140
+    },
+    {
+      "epoch": 0.0862533692722372,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.752634811401367,
+      "step": 160
+    },
+    {
+      "epoch": 0.09703504043126684,
+      "grad_norm": 0.4765625,
+      "learning_rate": 0.001,
+      "loss": 3.6034828186035157,
+      "step": 180
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 3.4787067413330077,
+      "step": 200
+    },
+    {
+      "epoch": 0.11859838274932614,
+      "grad_norm": 0.9296875,
+      "learning_rate": 0.001,
+      "loss": 3.3636749267578123,
+      "step": 220
+    },
+    {
+      "epoch": 0.1293800539083558,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.269498825073242,
+      "step": 240
+    },
+    {
+      "epoch": 0.14016172506738545,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 3.183236503601074,
+      "step": 260
+    },
+    {
+      "epoch": 0.1509433962264151,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.001,
+      "loss": 3.099096488952637,
+      "step": 280
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.8359375,
+      "learning_rate": 0.001,
+      "loss": 3.0244571685791017,
+      "step": 300
+    },
+    {
+      "epoch": 0.1725067385444744,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.001,
+      "loss": 2.9644407272338866,
+      "step": 320
+    },
+    {
+      "epoch": 0.18328840970350405,
+      "grad_norm": 0.83984375,
+      "learning_rate": 0.001,
+      "loss": 2.8989635467529298,
+      "step": 340
+    },
+    {
+      "epoch": 0.1940700808625337,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.001,
+      "loss": 2.851881980895996,
+      "step": 360
+    },
+    {
+      "epoch": 0.20485175202156333,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 2.790276527404785,
+      "step": 380
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.7615821838378904,
+      "step": 400
+    },
+    {
+      "epoch": 0.22641509433962265,
+      "grad_norm": 0.83203125,
+      "learning_rate": 0.001,
+      "loss": 2.7132925033569335,
+      "step": 420
+    },
+    {
+      "epoch": 0.2371967654986523,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.691997528076172,
+      "step": 440
+    },
+    {
+      "epoch": 0.24797843665768193,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.6550079345703126,
+      "step": 460
+    },
+    {
+      "epoch": 0.2587601078167116,
+      "grad_norm": 0.58984375,
+      "learning_rate": 0.001,
+      "loss": 2.6208978652954102,
+      "step": 480
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.001,
+      "loss": 2.593034362792969,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803234501347709,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.001,
+      "loss": 2.5671127319335936,
+      "step": 520
+    },
+    {
+      "epoch": 0.29110512129380056,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 2.5521316528320312,
+      "step": 540
+    },
+    {
+      "epoch": 0.3018867924528302,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.5245645523071287,
+      "step": 560
+    },
+    {
+      "epoch": 0.31266846361185985,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 2.5037845611572265,
+      "step": 580
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.74609375,
+      "learning_rate": 0.001,
+      "loss": 2.482448196411133,
+      "step": 600
+    },
+    {
+      "epoch": 0.33423180592991913,
+      "grad_norm": 1.078125,
+      "learning_rate": 0.001,
+      "loss": 2.4656560897827147,
+      "step": 620
+    },
+    {
+      "epoch": 0.3450134770889488,
+      "grad_norm": 0.5703125,
+      "learning_rate": 0.001,
+      "loss": 2.4476362228393556,
+      "step": 640
+    },
+    {
+      "epoch": 0.3557951482479784,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 2.4330963134765624,
+      "step": 660
+    },
+    {
+      "epoch": 0.3665768194070081,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.41370849609375,
+      "step": 680
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.397682952880859,
+      "step": 700
+    },
+    {
+      "epoch": 0.3881401617250674,
+      "grad_norm": 0.625,
+      "learning_rate": 0.001,
+      "loss": 2.3781137466430664,
+      "step": 720
+    },
+    {
+      "epoch": 0.39892183288409705,
+      "grad_norm": 0.64453125,
+      "learning_rate": 0.001,
+      "loss": 2.3710350036621093,
+      "step": 740
+    },
+    {
+      "epoch": 0.40970350404312667,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 2.3579540252685547,
+      "step": 760
+    },
+    {
+      "epoch": 0.42048517520215634,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 2.34680233001709,
+      "step": 780
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.84375,
+      "learning_rate": 0.001,
+      "loss": 2.325694274902344,
+      "step": 800
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1858486036070400.0,
+  "train_batch_size": 512,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-800/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea57eafa5d8d7c92fa44d42011d21fc8a9b019b0c1ee506d1d2f3d7c5eb1dccd
+size 4920

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dccd73cc6a3aecb5e560178ad16a064c49385bf4014d52925353f9eeae6561b6
+size 4011496

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b6aef066603ff9ebd99bd48a514475e4ab612501a1420d4023f90ea5a0b1cfd
+size 8074746

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d48b00fea613f19335b87fb622648d28d7f8900963d07ba43c75df2b4dc3d4e
+size 1064

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/trainer_state.json

@@ -0,0 +1,349 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.48517520215633425,
+  "eval_steps": 60000,
+  "global_step": 900,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01078167115902965,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.001,
+      "loss": 7.1676788330078125,
+      "step": 20
+    },
+    {
+      "epoch": 0.0215633423180593,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.001,
+      "loss": 5.993187332153321,
+      "step": 40
+    },
+    {
+      "epoch": 0.03234501347708895,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.001,
+      "loss": 5.496040725708008,
+      "step": 60
+    },
+    {
+      "epoch": 0.0431266846361186,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.001,
+      "loss": 4.934689331054687,
+      "step": 80
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.001,
+      "loss": 4.479576110839844,
+      "step": 100
+    },
+    {
+      "epoch": 0.0646900269541779,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.001,
+      "loss": 4.171505355834961,
+      "step": 120
+    },
+    {
+      "epoch": 0.07547169811320754,
+      "grad_norm": 0.55078125,
+      "learning_rate": 0.001,
+      "loss": 3.940719223022461,
+      "step": 140
+    },
+    {
+      "epoch": 0.0862533692722372,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.752634811401367,
+      "step": 160
+    },
+    {
+      "epoch": 0.09703504043126684,
+      "grad_norm": 0.4765625,
+      "learning_rate": 0.001,
+      "loss": 3.6034828186035157,
+      "step": 180
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 3.4787067413330077,
+      "step": 200
+    },
+    {
+      "epoch": 0.11859838274932614,
+      "grad_norm": 0.9296875,
+      "learning_rate": 0.001,
+      "loss": 3.3636749267578123,
+      "step": 220
+    },
+    {
+      "epoch": 0.1293800539083558,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.269498825073242,
+      "step": 240
+    },
+    {
+      "epoch": 0.14016172506738545,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.001,
+      "loss": 3.183236503601074,
+      "step": 260
+    },
+    {
+      "epoch": 0.1509433962264151,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.001,
+      "loss": 3.099096488952637,
+      "step": 280
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 0.8359375,
+      "learning_rate": 0.001,
+      "loss": 3.0244571685791017,
+      "step": 300
+    },
+    {
+      "epoch": 0.1725067385444744,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.001,
+      "loss": 2.9644407272338866,
+      "step": 320
+    },
+    {
+      "epoch": 0.18328840970350405,
+      "grad_norm": 0.83984375,
+      "learning_rate": 0.001,
+      "loss": 2.8989635467529298,
+      "step": 340
+    },
+    {
+      "epoch": 0.1940700808625337,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.001,
+      "loss": 2.851881980895996,
+      "step": 360
+    },
+    {
+      "epoch": 0.20485175202156333,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 2.790276527404785,
+      "step": 380
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.7615821838378904,
+      "step": 400
+    },
+    {
+      "epoch": 0.22641509433962265,
+      "grad_norm": 0.83203125,
+      "learning_rate": 0.001,
+      "loss": 2.7132925033569335,
+      "step": 420
+    },
+    {
+      "epoch": 0.2371967654986523,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.691997528076172,
+      "step": 440
+    },
+    {
+      "epoch": 0.24797843665768193,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.6550079345703126,
+      "step": 460
+    },
+    {
+      "epoch": 0.2587601078167116,
+      "grad_norm": 0.58984375,
+      "learning_rate": 0.001,
+      "loss": 2.6208978652954102,
+      "step": 480
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.001,
+      "loss": 2.593034362792969,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803234501347709,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.001,
+      "loss": 2.5671127319335936,
+      "step": 520
+    },
+    {
+      "epoch": 0.29110512129380056,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 2.5521316528320312,
+      "step": 540
+    },
+    {
+      "epoch": 0.3018867924528302,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.5245645523071287,
+      "step": 560
+    },
+    {
+      "epoch": 0.31266846361185985,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 2.5037845611572265,
+      "step": 580
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.74609375,
+      "learning_rate": 0.001,
+      "loss": 2.482448196411133,
+      "step": 600
+    },
+    {
+      "epoch": 0.33423180592991913,
+      "grad_norm": 1.078125,
+      "learning_rate": 0.001,
+      "loss": 2.4656560897827147,
+      "step": 620
+    },
+    {
+      "epoch": 0.3450134770889488,
+      "grad_norm": 0.5703125,
+      "learning_rate": 0.001,
+      "loss": 2.4476362228393556,
+      "step": 640
+    },
+    {
+      "epoch": 0.3557951482479784,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 2.4330963134765624,
+      "step": 660
+    },
+    {
+      "epoch": 0.3665768194070081,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.41370849609375,
+      "step": 680
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.001,
+      "loss": 2.397682952880859,
+      "step": 700
+    },
+    {
+      "epoch": 0.3881401617250674,
+      "grad_norm": 0.625,
+      "learning_rate": 0.001,
+      "loss": 2.3781137466430664,
+      "step": 720
+    },
+    {
+      "epoch": 0.39892183288409705,
+      "grad_norm": 0.64453125,
+      "learning_rate": 0.001,
+      "loss": 2.3710350036621093,
+      "step": 740
+    },
+    {
+      "epoch": 0.40970350404312667,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 2.3579540252685547,
+      "step": 760
+    },
+    {
+      "epoch": 0.42048517520215634,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 2.34680233001709,
+      "step": 780
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.84375,
+      "learning_rate": 0.001,
+      "loss": 2.325694274902344,
+      "step": 800
+    },
+    {
+      "epoch": 0.4420485175202156,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.001,
+      "loss": 2.32082576751709,
+      "step": 820
+    },
+    {
+      "epoch": 0.4528301886792453,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.001,
+      "loss": 2.3087087631225587,
+      "step": 840
+    },
+    {
+      "epoch": 0.4636118598382749,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.001,
+      "loss": 2.293583869934082,
+      "step": 860
+    },
+    {
+      "epoch": 0.4743935309973046,
+      "grad_norm": 0.78125,
+      "learning_rate": 0.001,
+      "loss": 2.283832550048828,
+      "step": 880
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.4765625,
+      "learning_rate": 0.001,
+      "loss": 2.272108268737793,
+      "step": 900
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2090796790579200.0,
+  "train_batch_size": 512,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-9L_run/checkpoint-900/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea57eafa5d8d7c92fa44d42011d21fc8a9b019b0c1ee506d1d2f3d7c5eb1dccd
+size 4920

zain/Activation/out/glu-silu-waleed10-9L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-9L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca7a808ba9383f2a1f20a31108f60a1dd105c32aefaf496bec84c001f010d893
+size 4011496

zain/Activation/out/glu-silu-waleed10-9L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-9L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea57eafa5d8d7c92fa44d42011d21fc8a9b019b0c1ee506d1d2f3d7c5eb1dccd
+size 4920

zain/Activation/out/glu-silu-waleed10-9L_run/training_log.jsonl

@@ -27,3 +27,27 @@
 {"step": 540, "epoch": 0.29110512129380056, "timestamp": 1786738947.30537, "loss": 2.5521316528320312, "grad_norm": 0.6875, "learning_rate": 0.001, "train/total_time_seconds": 168.4331082291901, "train/time_per_step_avg": 0.3093720481172204, "train/epoch_time_elapsed": 370.7744383215904, "train/estimated_remaining_minutes": 2.3913342526366494}
 {"step": 560, "epoch": 0.3018867924528302, "timestamp": 1786738961.1207943, "loss": 2.5245645523071287, "grad_norm": 0.59375, "learning_rate": 0.001, "train/total_time_seconds": 174.73105559498072, "train/time_per_step_avg": 0.31588548950850964, "train/epoch_time_elapsed": 384.58986269682646, "train/estimated_remaining_minutes": 2.288144775648557}
 {"step": 580, "epoch": 0.31266846361185985, "timestamp": 1786738974.9993198, "loss": 2.5037845611572265, "grad_norm": 0.58203125, "learning_rate": 0.001, "train/total_time_seconds": 181.1427909322083, "train/time_per_step_avg": 0.31603880763053893, "train/epoch_time_elapsed": 398.4683878980577, "train/estimated_remaining_minutes": 2.1862060974576862}
+{"step": 600, "epoch": 0.32345013477088946, "timestamp": 1786738988.9234128, "loss": 2.482448196411133, "grad_norm": 0.74609375, "learning_rate": 0.001, "train/total_time_seconds": 187.51905519142747, "train/time_per_step_avg": 0.31568161509931086, "train/epoch_time_elapsed": 412.392480712384, "train/estimated_remaining_minutes": 2.083545057682527}
+{"step": 620, "epoch": 0.33423180592991913, "timestamp": 1786739002.8951075, "loss": 2.4656560897827147, "grad_norm": 1.078125, "learning_rate": 0.001, "train/total_time_seconds": 193.93395326659083, "train/time_per_step_avg": 0.3186231619864702, "train/epoch_time_elapsed": 426.3641758598387, "train/estimated_remaining_minutes": 1.9810457591748525}
+{"step": 640, "epoch": 0.3450134770889488, "timestamp": 1786739016.7006578, "loss": 2.4476362228393556, "grad_norm": 0.5703125, "learning_rate": 0.001, "train/total_time_seconds": 200.25969551876187, "train/time_per_step_avg": 0.3182658728957176, "train/epoch_time_elapsed": 440.16972568258643, "train/estimated_remaining_minutes": 1.8774346454883926}
+{"step": 660, "epoch": 0.3557951482479784, "timestamp": 1786739030.8592184, "loss": 2.4330963134765624, "grad_norm": 0.6953125, "learning_rate": 0.001, "train/total_time_seconds": 206.61607265472412, "train/time_per_step_avg": 0.31885017059743404, "train/epoch_time_elapsed": 454.32828671857715, "train/estimated_remaining_minutes": 1.7739763813789446}
+{"step": 680, "epoch": 0.3665768194070081, "timestamp": 1786739044.5286462, "loss": 2.41370849609375, "grad_norm": 0.66796875, "learning_rate": 0.001, "train/total_time_seconds": 212.6340763270855, "train/time_per_step_avg": 0.31491285394877194, "train/epoch_time_elapsed": 467.99771436303854, "train/estimated_remaining_minutes": 1.6677182457026314}
+{"step": 700, "epoch": 0.37735849056603776, "timestamp": 1786739058.5081441, "loss": 2.397682952880859, "grad_norm": 0.80078125, "learning_rate": 0.001, "train/total_time_seconds": 219.04045253619552, "train/time_per_step_avg": 0.3152139734476805, "train/epoch_time_elapsed": 481.9772127084434, "train/estimated_remaining_minutes": 1.564574660972825}
+{"step": 720, "epoch": 0.3881401617250674, "timestamp": 1786739072.5935593, "loss": 2.3781137466430664, "grad_norm": 0.625, "learning_rate": 0.001, "train/total_time_seconds": 225.45517436042428, "train/time_per_step_avg": 0.31521221093833446, "train/epoch_time_elapsed": 496.062627337873, "train/estimated_remaining_minutes": 1.4612835375212685}
+{"step": 740, "epoch": 0.39892183288409705, "timestamp": 1786739086.666067, "loss": 2.3710350036621093, "grad_norm": 0.64453125, "learning_rate": 0.001, "train/total_time_seconds": 231.97142902761698, "train/time_per_step_avg": 0.31711733508855106, "train/epoch_time_elapsed": 510.135135024786, "train/estimated_remaining_minutes": 1.358391251062622}
+{"step": 760, "epoch": 0.40970350404312667, "timestamp": 1786739100.4253023, "loss": 2.3579540252685547, "grad_norm": 0.7265625, "learning_rate": 0.001, "train/total_time_seconds": 238.24562624841928, "train/time_per_step_avg": 0.31629553593695164, "train/epoch_time_elapsed": 523.8943699933589, "train/estimated_remaining_minutes": 1.2539243486758909}
+{"step": 780, "epoch": 0.42048517520215634, "timestamp": 1786739114.2904058, "loss": 2.34680233001709, "grad_norm": 0.6484375, "learning_rate": 0.001, "train/total_time_seconds": 244.58184621855617, "train/time_per_step_avg": 0.31947769891470673, "train/epoch_time_elapsed": 537.7594740577042, "train/estimated_remaining_minutes": 1.1497437215402213}
+{"step": 800, "epoch": 0.431266846361186, "timestamp": 1786739127.9673479, "loss": 2.325694274902344, "grad_norm": 0.84375, "learning_rate": 0.001, "train/total_time_seconds": 250.7181674055755, "train/time_per_step_avg": 0.31677714869379997, "train/epoch_time_elapsed": 551.4364160113037, "train/estimated_remaining_minutes": 1.0446590308565649}
+{"step": 820, "epoch": 0.4420485175202156, "timestamp": 1786739141.9621968, "loss": 2.32082576751709, "grad_norm": 0.62109375, "learning_rate": 0.001, "train/total_time_seconds": 257.1595130302012, "train/time_per_step_avg": 0.31704338669776916, "train/epoch_time_elapsed": 565.4312646724284, "train/estimated_remaining_minutes": 0.940827486695858}
+{"step": 840, "epoch": 0.4528301886792453, "timestamp": 1786739155.7160172, "loss": 2.3087087631225587, "grad_norm": 0.78515625, "learning_rate": 0.001, "train/total_time_seconds": 263.3063704930246, "train/time_per_step_avg": 0.3133494146540761, "train/epoch_time_elapsed": 579.185085516423, "train/estimated_remaining_minutes": 0.8358932396603955}
+{"step": 860, "epoch": 0.4636118598382749, "timestamp": 1786739169.8307574, "loss": 2.293583869934082, "grad_norm": 0.59765625, "learning_rate": 0.001, "train/total_time_seconds": 269.81821148842573, "train/time_per_step_avg": 0.31572585240006446, "train/epoch_time_elapsed": 593.29982592538, "train/estimated_remaining_minutes": 0.7320649148910775}
+{"step": 880, "epoch": 0.4743935309973046, "timestamp": 1786739183.8156111, "loss": 2.283832550048828, "grad_norm": 0.78125, "learning_rate": 0.001, "train/total_time_seconds": 276.24523336067796, "train/time_per_step_avg": 0.3166338714212179, "train/epoch_time_elapsed": 607.2846792638302, "train/estimated_remaining_minutes": 0.6278300758197226}
+{"step": 900, "epoch": 0.48517520215633425, "timestamp": 1786739197.5198941, "loss": 2.272108268737793, "grad_norm": 0.4765625, "learning_rate": 0.001, "train/total_time_seconds": 282.42855532839894, "train/time_per_step_avg": 0.3171038792282343, "train/epoch_time_elapsed": 620.9889623522758, "train/estimated_remaining_minutes": 0.5230158432007388}
+{"step": 920, "epoch": 0.49595687331536387, "timestamp": 1786739211.5306232, "loss": 2.263412284851074, "grad_norm": 0.64453125, "learning_rate": 0.001, "train/total_time_seconds": 288.8141548484564, "train/time_per_step_avg": 0.31654641818255186, "train/epoch_time_elapsed": 634.999691568315, "train/estimated_remaining_minutes": 0.4185712389108064}
+{"step": 940, "epoch": 0.5067385444743935, "timestamp": 1786739225.3523896, "loss": 2.261708450317383, "grad_norm": 0.63671875, "learning_rate": 0.001, "train/total_time_seconds": 295.039514105767, "train/time_per_step_avg": 0.31733143612742426, "train/epoch_time_elapsed": 648.8214579708874, "train/estimated_remaining_minutes": 0.3138718235167734}
+{"step": 960, "epoch": 0.5175202156334232, "timestamp": 1786739239.3667402, "loss": 2.255706214904785, "grad_norm": 0.6484375, "learning_rate": 0.001, "train/total_time_seconds": 301.5321364104748, "train/time_per_step_avg": 0.31713924922049047, "train/epoch_time_elapsed": 662.8358082510531, "train/estimated_remaining_minutes": 0.2093973169517186}
+{"step": 980, "epoch": 0.5283018867924528, "timestamp": 1786739253.1727662, "loss": 2.24764461517334, "grad_norm": 0.6015625, "learning_rate": 0.001, "train/total_time_seconds": 307.84208834543824, "train/time_per_step_avg": 0.31596854984760286, "train/epoch_time_elapsed": 676.6418348029256, "train/estimated_remaining_minutes": 0.1047081933147749}
+{"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786739266.9835904, "loss": 2.2394777297973634, "grad_norm": 0.90234375, "learning_rate": 0.001, "train/total_time_seconds": 314.0651192739606, "train/time_per_step_avg": 0.31636563945561647, "train/epoch_time_elapsed": 690.452658995986, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786739276.111055, "eval_loss": 2.241689443588257, "eval_runtime": 9.1259, "eval_samples_per_second": 1043.957, "eval_steps_per_second": 2.082, "train/total_time_seconds": 314.0651192739606, "train/time_per_step_avg": 0.31636563945561647, "train/epoch_time_elapsed": 699.5801220946014, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786739276.1596305, "train_runtime": 700.4885, "train_samples_per_second": 730.918, "train_steps_per_second": 1.428, "total_flos": 2323107545088000.0, "train_loss": 2.996279800415039, "train/total_time_seconds": 314.0651192739606, "train/time_per_step_avg": 0.31636563945561647, "train/epoch_time_elapsed": 699.6286974214017, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786739285.3598566, "eval_loss": 2.241689443588257, "eval_runtime": 9.1977, "eval_samples_per_second": 1035.797, "eval_steps_per_second": 2.066, "train/total_time_seconds": 314.0651192739606, "train/time_per_step_avg": 0.31636563945561647, "train/epoch_time_elapsed": 708.828924253583, "train/estimated_remaining_minutes": 0.0}

zain/Activation/out/glu-waleed10-100L_run/checkpoint-1000/config.json

@@ -32,5 +32,5 @@
   "transformers_version": "5.16.0.dev0",
   "use_cache": false,
   "vocab_size": 4096,
-  "waleed_beta": 10.0
+  "waleed_beta": 4
 }

zain/Activation/out/glu-waleed10-100L_run/checkpoint-1000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3d885425d2f6f8444afc96f98b006666e66a535c1b37bba6e5250e2790bd7a4
+oid sha256:daf7093bbbb738c647c2d150dc74706b989e17e618dd3a0a7dfa24ef8b4e91f8
 size 33967272

zain/Activation/out/glu-waleed10-100L_run/checkpoint-1000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54c90fbb92c827be9c8fe6279e57a7359fedfd865f28d79f2d83c17b3ca24835
+oid sha256:8272701795dc0eab081dd863448ee289fd413f37cf4113de6819bbed4d6cd62b
 size 68504996

zain/Activation/out/glu-waleed10-100L_run/checkpoint-1000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
 size 14244

zain/Activation/out/glu-waleed10-100L_run/checkpoint-1000/trainer_state.json

@@ -3,7 +3,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 0.06740815638692282,
-  "eval_steps": 50,
+  "eval_steps": 2498,
   "global_step": 1000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -13,510 +13,350 @@
       "epoch": 0.0013481631277384564,
       "grad_norm": 2.125,
       "learning_rate": 2.66e-05,
-      "loss": 8.29942626953125,
+      "loss": 8.29943389892578,
       "step": 20
     },
     {
       "epoch": 0.002696326255476913,
       "grad_norm": 1.3046875,
       "learning_rate": 5.46e-05,
-      "loss": 8.036360168457032,
+      "loss": 8.03638916015625,
       "step": 40
     },
-    {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.832070827484131,
-      "eval_runtime": 15.4822,
-      "eval_samples_per_second": 615.352,
-      "eval_steps_per_second": 0.646,
-      "step": 50
-    },
     {
       "epoch": 0.004044489383215369,
       "grad_norm": 1.2734375,
       "learning_rate": 8.259999999999999e-05,
-      "loss": 7.834564208984375,
+      "loss": 7.834577941894532,
       "step": 60
     },
     {
       "epoch": 0.005392652510953826,
       "grad_norm": 1.265625,
       "learning_rate": 0.0001106,
-      "loss": 7.5718849182128904,
+      "loss": 7.5719154357910154,
       "step": 80
     },
     {
       "epoch": 0.006740815638692282,
       "grad_norm": 1.1953125,
       "learning_rate": 0.0001386,
-      "loss": 7.2372383117675785,
-      "step": 100
-    },
-    {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 7.0443572998046875,
-      "eval_runtime": 15.5098,
-      "eval_samples_per_second": 614.257,
-      "eval_steps_per_second": 0.645,
+      "loss": 7.23726806640625,
       "step": 100
     },
     {
       "epoch": 0.008088978766430738,
       "grad_norm": 1.1484375,
       "learning_rate": 0.00016659999999999998,
-      "loss": 6.864932250976563,
+      "loss": 6.864923095703125,
       "step": 120
     },
     {
       "epoch": 0.009437141894169195,
       "grad_norm": 1.0234375,
       "learning_rate": 0.00019460000000000001,
-      "loss": 6.509996032714843,
+      "loss": 6.509650421142578,
       "step": 140
     },
-    {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 6.1924004554748535,
-      "eval_runtime": 15.382,
-      "eval_samples_per_second": 619.361,
-      "eval_steps_per_second": 0.65,
-      "step": 150
-    },
     {
       "epoch": 0.010785305021907651,
-      "grad_norm": 0.96484375,
+      "grad_norm": 1.03125,
       "learning_rate": 0.0002226,
-      "loss": 6.203652191162109,
+      "loss": 6.204092788696289,
       "step": 160
     },
     {
       "epoch": 0.012133468149646108,
-      "grad_norm": 1.0234375,
+      "grad_norm": 0.87109375,
       "learning_rate": 0.00025059999999999997,
-      "loss": 5.938727951049804,
+      "loss": 5.9408447265625,
       "step": 180
     },
     {
       "epoch": 0.013481631277384564,
-      "grad_norm": 0.90234375,
+      "grad_norm": 0.76171875,
       "learning_rate": 0.0002786,
-      "loss": 5.677180480957031,
-      "step": 200
-    },
-    {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.540520668029785,
-      "eval_runtime": 15.6734,
-      "eval_samples_per_second": 607.844,
-      "eval_steps_per_second": 0.638,
+      "loss": 5.677399826049805,
       "step": 200
     },
     {
       "epoch": 0.01482979440512302,
-      "grad_norm": 1.5078125,
+      "grad_norm": 0.68359375,
       "learning_rate": 0.00030659999999999997,
-      "loss": 5.418574523925781,
+      "loss": 5.419148254394531,
       "step": 220
     },
     {
       "epoch": 0.016177957532861477,
-      "grad_norm": 0.7109375,
+      "grad_norm": 0.65625,
       "learning_rate": 0.0003346,
-      "loss": 5.1351776123046875,
+      "loss": 5.138995742797851,
       "step": 240
     },
-    {
-      "epoch": 0.016852039096730706,
-      "eval_loss": 4.857514381408691,
-      "eval_runtime": 15.4191,
-      "eval_samples_per_second": 617.872,
-      "eval_steps_per_second": 0.649,
-      "step": 250
-    },
     {
       "epoch": 0.01752612066059993,
-      "grad_norm": 0.875,
+      "grad_norm": 1.265625,
       "learning_rate": 0.00036260000000000003,
-      "loss": 4.880479431152343,
+      "loss": 4.88677749633789,
       "step": 260
     },
     {
       "epoch": 0.01887428378833839,
-      "grad_norm": 0.875,
+      "grad_norm": 0.80078125,
       "learning_rate": 0.0003906,
-      "loss": 4.645774841308594,
+      "loss": 4.653627014160156,
       "step": 280
     },
     {
       "epoch": 0.020222446916076844,
-      "grad_norm": 0.6640625,
+      "grad_norm": 0.64453125,
       "learning_rate": 0.0004186,
-      "loss": 4.451469421386719,
-      "step": 300
-    },
-    {
-      "epoch": 0.020222446916076844,
-      "eval_loss": 4.3544721603393555,
-      "eval_runtime": 15.3937,
-      "eval_samples_per_second": 618.891,
-      "eval_steps_per_second": 0.65,
+      "loss": 4.45317268371582,
       "step": 300
     },
     {
       "epoch": 0.021570610043815303,
-      "grad_norm": 0.609375,
+      "grad_norm": 1.1484375,
       "learning_rate": 0.0004466,
-      "loss": 4.269550323486328,
+      "loss": 4.271414947509766,
       "step": 320
     },
     {
       "epoch": 0.022918773171553757,
-      "grad_norm": 0.50390625,
+      "grad_norm": 0.52734375,
       "learning_rate": 0.00047460000000000004,
-      "loss": 4.115790176391601,
+      "loss": 4.136629104614258,
       "step": 340
     },
-    {
-      "epoch": 0.023592854735422986,
-      "eval_loss": 3.9874157905578613,
-      "eval_runtime": 15.3586,
-      "eval_samples_per_second": 620.304,
-      "eval_steps_per_second": 0.651,
-      "step": 350
-    },
     {
       "epoch": 0.024266936299292215,
-      "grad_norm": 0.8828125,
+      "grad_norm": 0.5390625,
       "learning_rate": 0.0005026,
-      "loss": 3.9873260498046874,
+      "loss": 4.001481246948242,
       "step": 360
     },
     {
       "epoch": 0.02561509942703067,
-      "grad_norm": 0.65234375,
+      "grad_norm": 0.75,
       "learning_rate": 0.0005306,
-      "loss": 3.873243713378906,
+      "loss": 3.8803211212158204,
       "step": 380
     },
     {
       "epoch": 0.026963262554769128,
-      "grad_norm": 0.50390625,
+      "grad_norm": 0.546875,
       "learning_rate": 0.0005586,
-      "loss": 3.7449169158935547,
-      "step": 400
-    },
-    {
-      "epoch": 0.026963262554769128,
-      "eval_loss": 3.713116407394409,
-      "eval_runtime": 15.3868,
-      "eval_samples_per_second": 619.166,
-      "eval_steps_per_second": 0.65,
+      "loss": 3.755691146850586,
       "step": 400
     },
     {
       "epoch": 0.028311425682507583,
-      "grad_norm": 0.53125,
+      "grad_norm": 0.5546875,
       "learning_rate": 0.0005866,
-      "loss": 3.653180694580078,
+      "loss": 3.6663139343261717,
       "step": 420
     },
     {
       "epoch": 0.02965958881024604,
-      "grad_norm": 0.59375,
+      "grad_norm": 0.56640625,
       "learning_rate": 0.0006146,
-      "loss": 3.599313735961914,
+      "loss": 3.6138153076171875,
       "step": 440
     },
-    {
-      "epoch": 0.030333670374115267,
-      "eval_loss": 3.5332272052764893,
-      "eval_runtime": 15.3821,
-      "eval_samples_per_second": 619.355,
-      "eval_steps_per_second": 0.65,
-      "step": 450
-    },
     {
       "epoch": 0.031007751937984496,
-      "grad_norm": 0.57421875,
+      "grad_norm": 0.435546875,
       "learning_rate": 0.0006426,
-      "loss": 3.5110404968261717,
+      "loss": 3.514398193359375,
       "step": 460
     },
     {
       "epoch": 0.032355915065722954,
-      "grad_norm": 0.671875,
+      "grad_norm": 0.5703125,
       "learning_rate": 0.0006705999999999999,
-      "loss": 3.4395523071289062,
+      "loss": 3.4464744567871093,
       "step": 480
     },
     {
       "epoch": 0.03370407819346141,
-      "grad_norm": 0.53125,
+      "grad_norm": 0.48828125,
       "learning_rate": 0.0006986,
-      "loss": 3.3704925537109376,
-      "step": 500
-    },
-    {
-      "epoch": 0.03370407819346141,
-      "eval_loss": 3.340529680252075,
-      "eval_runtime": 15.5207,
-      "eval_samples_per_second": 613.824,
-      "eval_steps_per_second": 0.644,
+      "loss": 3.377140426635742,
       "step": 500
     },
     {
       "epoch": 0.03505224132119986,
-      "grad_norm": 0.49609375,
+      "grad_norm": 0.5078125,
       "learning_rate": 0.0007,
-      "loss": 3.3107833862304688,
+      "loss": 3.3188812255859377,
       "step": 520
     },
     {
       "epoch": 0.03640040444893832,
-      "grad_norm": 0.51171875,
+      "grad_norm": 0.56640625,
       "learning_rate": 0.0007,
-      "loss": 3.2391616821289064,
+      "loss": 3.2469013214111326,
       "step": 540
     },
-    {
-      "epoch": 0.03707448601280755,
-      "eval_loss": 3.2028048038482666,
-      "eval_runtime": 15.6434,
-      "eval_samples_per_second": 609.013,
-      "eval_steps_per_second": 0.639,
-      "step": 550
-    },
     {
       "epoch": 0.03774856757667678,
-      "grad_norm": 0.466796875,
+      "grad_norm": 0.51171875,
       "learning_rate": 0.0007,
-      "loss": 3.1940614700317385,
+      "loss": 3.199608039855957,
       "step": 560
     },
     {
       "epoch": 0.03909673070441524,
-      "grad_norm": 0.4921875,
+      "grad_norm": 0.51953125,
       "learning_rate": 0.0007,
-      "loss": 3.141546440124512,
+      "loss": 3.1469419479370115,
       "step": 580
     },
     {
       "epoch": 0.04044489383215369,
-      "grad_norm": 0.4609375,
+      "grad_norm": 0.4765625,
       "learning_rate": 0.0007,
-      "loss": 3.1114213943481444,
-      "step": 600
-    },
-    {
-      "epoch": 0.04044489383215369,
-      "eval_loss": 3.0854389667510986,
-      "eval_runtime": 15.4041,
-      "eval_samples_per_second": 618.47,
-      "eval_steps_per_second": 0.649,
+      "loss": 3.117013168334961,
       "step": 600
     },
     {
       "epoch": 0.04179305695989215,
-      "grad_norm": 0.5546875,
+      "grad_norm": 0.50390625,
       "learning_rate": 0.0007,
-      "loss": 3.0545623779296873,
+      "loss": 3.0596269607543944,
       "step": 620
     },
     {
       "epoch": 0.043141220087630605,
-      "grad_norm": 0.50390625,
+      "grad_norm": 0.451171875,
       "learning_rate": 0.0007,
-      "loss": 3.0375244140625,
+      "loss": 3.04007568359375,
       "step": 640
     },
-    {
-      "epoch": 0.043815301651499834,
-      "eval_loss": 2.997983455657959,
-      "eval_runtime": 15.3695,
-      "eval_samples_per_second": 619.864,
-      "eval_steps_per_second": 0.651,
-      "step": 650
-    },
     {
       "epoch": 0.044489383215369056,
-      "grad_norm": 0.470703125,
+      "grad_norm": 0.4765625,
       "learning_rate": 0.0007,
-      "loss": 2.9873325347900392,
+      "loss": 2.9884712219238283,
       "step": 660
     },
     {
       "epoch": 0.045837546343107514,
-      "grad_norm": 0.462890625,
+      "grad_norm": 0.455078125,
       "learning_rate": 0.0007,
-      "loss": 2.9647268295288085,
+      "loss": 2.965327835083008,
       "step": 680
     },
     {
       "epoch": 0.04718570947084597,
-      "grad_norm": 0.435546875,
+      "grad_norm": 0.42578125,
       "learning_rate": 0.0007,
-      "loss": 2.924059295654297,
-      "step": 700
-    },
-    {
-      "epoch": 0.04718570947084597,
-      "eval_loss": 2.9152462482452393,
-      "eval_runtime": 15.4441,
-      "eval_samples_per_second": 616.871,
-      "eval_steps_per_second": 0.647,
+      "loss": 2.9250982284545897,
       "step": 700
     },
     {
       "epoch": 0.04853387259858443,
-      "grad_norm": 0.447265625,
+      "grad_norm": 0.42578125,
       "learning_rate": 0.0007,
-      "loss": 2.895351600646973,
+      "loss": 2.895215606689453,
       "step": 720
     },
     {
       "epoch": 0.04988203572632288,
-      "grad_norm": 0.4453125,
+      "grad_norm": 0.490234375,
       "learning_rate": 0.0007,
-      "loss": 2.861538314819336,
+      "loss": 2.859494209289551,
       "step": 740
     },
-    {
-      "epoch": 0.05055611729019211,
-      "eval_loss": 2.849472761154175,
-      "eval_runtime": 15.4441,
-      "eval_samples_per_second": 616.868,
-      "eval_steps_per_second": 0.647,
-      "step": 750
-    },
     {
       "epoch": 0.05123019885406134,
-      "grad_norm": 0.478515625,
+      "grad_norm": 0.474609375,
       "learning_rate": 0.0007,
-      "loss": 2.848223114013672,
+      "loss": 2.8466249465942384,
       "step": 760
     },
     {
       "epoch": 0.0525783619817998,
-      "grad_norm": 0.498046875,
+      "grad_norm": 0.4453125,
       "learning_rate": 0.0007,
-      "loss": 2.8442684173583985,
+      "loss": 2.843080520629883,
       "step": 780
     },
     {
       "epoch": 0.053926525109538256,
-      "grad_norm": 0.421875,
+      "grad_norm": 0.4296875,
       "learning_rate": 0.0007,
-      "loss": 2.802508735656738,
-      "step": 800
-    },
-    {
-      "epoch": 0.053926525109538256,
-      "eval_loss": 2.7919585704803467,
-      "eval_runtime": 15.379,
-      "eval_samples_per_second": 619.481,
-      "eval_steps_per_second": 0.65,
+      "loss": 2.8001882553100588,
       "step": 800
     },
     {
       "epoch": 0.05527468823727671,
-      "grad_norm": 0.46484375,
+      "grad_norm": 0.4296875,
       "learning_rate": 0.0007,
-      "loss": 2.791913604736328,
+      "loss": 2.790140151977539,
       "step": 820
     },
     {
       "epoch": 0.056622851365015166,
       "grad_norm": 0.455078125,
       "learning_rate": 0.0007,
-      "loss": 2.764504241943359,
+      "loss": 2.762985420227051,
       "step": 840
     },
-    {
-      "epoch": 0.057296932928884395,
-      "eval_loss": 2.733065366744995,
-      "eval_runtime": 15.5198,
-      "eval_samples_per_second": 613.863,
-      "eval_steps_per_second": 0.644,
-      "step": 850
-    },
     {
       "epoch": 0.057971014492753624,
-      "grad_norm": 0.50390625,
+      "grad_norm": 0.435546875,
       "learning_rate": 0.0007,
-      "loss": 2.723073387145996,
+      "loss": 2.721993637084961,
       "step": 860
     },
     {
       "epoch": 0.05931917762049208,
-      "grad_norm": 0.419921875,
+      "grad_norm": 0.42578125,
       "learning_rate": 0.0007,
-      "loss": 2.700708198547363,
+      "loss": 2.699744987487793,
       "step": 880
     },
     {
       "epoch": 0.06066734074823053,
-      "grad_norm": 0.46484375,
+      "grad_norm": 0.439453125,
       "learning_rate": 0.0007,
-      "loss": 2.6873470306396485,
-      "step": 900
-    },
-    {
-      "epoch": 0.06066734074823053,
-      "eval_loss": 2.682546615600586,
-      "eval_runtime": 15.4179,
-      "eval_samples_per_second": 617.919,
-      "eval_steps_per_second": 0.649,
+      "loss": 2.689156150817871,
       "step": 900
     },
     {
       "epoch": 0.06201550387596899,
-      "grad_norm": 0.478515625,
+      "grad_norm": 0.4921875,
       "learning_rate": 0.0007,
-      "loss": 2.663492965698242,
+      "loss": 2.6624650955200195,
       "step": 920
     },
     {
       "epoch": 0.06336366700370745,
-      "grad_norm": 0.46875,
+      "grad_norm": 0.44140625,
       "learning_rate": 0.0007,
-      "loss": 2.653563690185547,
+      "loss": 2.6517080307006835,
       "step": 940
     },
-    {
-      "epoch": 0.06403774856757667,
-      "eval_loss": 2.636122941970825,
-      "eval_runtime": 15.6175,
-      "eval_samples_per_second": 610.019,
-      "eval_steps_per_second": 0.64,
-      "step": 950
-    },
     {
       "epoch": 0.06471183013144591,
-      "grad_norm": 0.455078125,
+      "grad_norm": 0.435546875,
       "learning_rate": 0.0007,
-      "loss": 2.628633499145508,
+      "loss": 2.6265542984008787,
       "step": 960
     },
     {
       "epoch": 0.06605999325918437,
-      "grad_norm": 0.44140625,
+      "grad_norm": 0.408203125,
       "learning_rate": 0.0007,
-      "loss": 2.6126628875732423,
+      "loss": 2.612698554992676,
       "step": 980
     },
     {
       "epoch": 0.06740815638692282,
-      "grad_norm": 0.4453125,
+      "grad_norm": 0.45703125,
       "learning_rate": 0.0007,
-      "loss": 2.6122547149658204,
-      "step": 1000
-    },
-    {
-      "epoch": 0.06740815638692282,
-      "eval_loss": 2.5941500663757324,
-      "eval_runtime": 15.5006,
-      "eval_samples_per_second": 614.621,
-      "eval_steps_per_second": 0.645,
+      "loss": 2.6142263412475586,
       "step": 1000
     }
   ],

zain/Activation/out/glu-waleed10-100L_run/checkpoint-1000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95042bce3268a58e39bf6898ae4dc3d7f41ae0258221813fdf6720b6f7599e44
+oid sha256:9a5eecd63c938d7d339a59fa838b1d3989a5cd93b45f48430293465bdb710244
 size 4920

zain/Activation/out/glu-waleed10-100L_run/checkpoint-300/config.json

@@ -32,5 +32,5 @@
   "transformers_version": "5.16.0.dev0",
   "use_cache": false,
   "vocab_size": 4096,
-  "waleed_beta": 10.0
+  "waleed_beta": 4
 }

zain/Activation/out/glu-waleed10-100L_run/checkpoint-300/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:caf3f6fbffbc2b59a936eeb09e88880be08b51a7cb1440b7b0c73c7f73800b67
+oid sha256:ce2ae685b6c3597ed8b24edb3150cd5902060cb0cefebc999786ec138b4c5dc8
 size 33967272