Auto upload zain 2026-08-12T23:41:37.855505

zain/Activation/out/glu-linear-100L_run/checkpoint-100/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "linear",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:152c1af9f3beaf4b02a6df8c07011aebb32584e8b42d00710bac48a9344d7b86
+size 33967272

zain/Activation/out/glu-linear-100L_run/checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8c98193087c4e9aa811942407560bed58f15aeea607ffb9bbf94f341ffbff6d
+size 68504996

zain/Activation/out/glu-linear-100L_run/checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-linear-100L_run/checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3001486ba00eb51d89ef078fe70e7e37535cd4e2311d7dd589266d045c9fb892
+size 1064

zain/Activation/out/glu-linear-100L_run/checkpoint-100/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-100/trainer_state.json

@@ -0,0 +1,69 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.006740815638692282,
+  "eval_steps": 2498,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29943389892578,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3046875,
+      "learning_rate": 5.46e-05,
+      "loss": 8.036395263671874,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.834572601318359,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5718994140625,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.237230682373047,
+      "step": 100
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 322628380262400.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f7f80cd089898db6bc7e5512322b4c5393daccc1c74b5f3c88feb826b29d13
+size 4920

zain/Activation/out/glu-linear-100L_run/checkpoint-200/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "linear",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a7ccf4fb256765315b3df6eb95185027c9e2468f984cce1d1655dd26a3d008f
+size 33967272

zain/Activation/out/glu-linear-100L_run/checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1df3fc315ee0b22aca95966976a086564fb6d25030c451819ddd8f5e1af9e571
+size 68504996

zain/Activation/out/glu-linear-100L_run/checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-linear-100L_run/checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28668eba09299968c820a8f82f4368de9f79bd22806c6be52cebe6b8700b6584
+size 1064

zain/Activation/out/glu-linear-100L_run/checkpoint-200/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-200/trainer_state.json

@@ -0,0 +1,104 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.013481631277384564,
+  "eval_steps": 2498,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29943389892578,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3046875,
+      "learning_rate": 5.46e-05,
+      "loss": 8.036395263671874,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.834572601318359,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5718994140625,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.237230682373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1484375,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.864912414550782,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.509800720214844,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.0002226,
+      "loss": 6.20410041809082,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.875,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 5.940516662597656,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.0002786,
+      "loss": 5.676180267333985,
+      "step": 200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 645256760524800.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f7f80cd089898db6bc7e5512322b4c5393daccc1c74b5f3c88feb826b29d13
+size 4920

zain/Activation/out/glu-linear-100L_run/checkpoint-300/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "linear",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-300/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbae4d0be5731789d638b853bbf20f2257f15f19fc98a982f986536f221b807a
+size 33967272

zain/Activation/out/glu-linear-100L_run/checkpoint-300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ffbf21a1318991c6482b194c5cc537f46dc40138f9f174660faaa7fbc2bac6e
+size 68504996

zain/Activation/out/glu-linear-100L_run/checkpoint-300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-linear-100L_run/checkpoint-300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:293635f74766f4fcae9882776529c6446698e9b875f847aa519f83843c9a249f
+size 1064

zain/Activation/out/glu-linear-100L_run/checkpoint-300/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-300/trainer_state.json

@@ -0,0 +1,139 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.020222446916076844,
+  "eval_steps": 2498,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29943389892578,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3046875,
+      "learning_rate": 5.46e-05,
+      "loss": 8.036395263671874,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.834572601318359,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5718994140625,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.237230682373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1484375,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.864912414550782,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.509800720214844,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.0002226,
+      "loss": 6.20410041809082,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.875,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 5.940516662597656,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.0002786,
+      "loss": 5.676180267333985,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.412886428833008,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.81640625,
+      "learning_rate": 0.0003346,
+      "loss": 5.135355377197266,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 4.882264709472656,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.0003906,
+      "loss": 4.6449634552001955,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0004186,
+      "loss": 4.446089553833008,
+      "step": 300
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 967885140787200.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f7f80cd089898db6bc7e5512322b4c5393daccc1c74b5f3c88feb826b29d13
+size 4920

zain/Activation/out/glu-linear-100L_run/checkpoint-400/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "linear",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0300b0101a0c3d0b8a380af7209f56ea817ff08a8e16464d9b938e8c4ccf1dc1
+size 33967272

zain/Activation/out/glu-linear-100L_run/checkpoint-400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c51681be3ae4787fc010cbff61d0c7d5fa689b1311507a07f600a513bebd1f4
+size 68504996

zain/Activation/out/glu-linear-100L_run/checkpoint-400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-linear-100L_run/checkpoint-400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e47d7e6a52e25825fc169e8ad3b4276c59dc20b6f1c01729d3fa2f5a57f163a
+size 1064

zain/Activation/out/glu-linear-100L_run/checkpoint-400/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-400/trainer_state.json

@@ -0,0 +1,174 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.026963262554769128,
+  "eval_steps": 2498,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29943389892578,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3046875,
+      "learning_rate": 5.46e-05,
+      "loss": 8.036395263671874,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.834572601318359,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5718994140625,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.237230682373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1484375,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.864912414550782,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.509800720214844,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.0002226,
+      "loss": 6.20410041809082,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.875,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 5.940516662597656,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.0002786,
+      "loss": 5.676180267333985,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.412886428833008,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.81640625,
+      "learning_rate": 0.0003346,
+      "loss": 5.135355377197266,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 4.882264709472656,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.0003906,
+      "loss": 4.6449634552001955,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0004186,
+      "loss": 4.446089553833008,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.0004466,
+      "loss": 4.2602802276611325,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.108924484252929,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.0005026,
+      "loss": 3.984059143066406,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.0005306,
+      "loss": 3.864672088623047,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.0005586,
+      "loss": 3.741139602661133,
+      "step": 400
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1290513521049600.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f7f80cd089898db6bc7e5512322b4c5393daccc1c74b5f3c88feb826b29d13
+size 4920

zain/Activation/out/glu-linear-100L_run/checkpoint-500/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "linear",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad97c100693150d69a3e78d19f78aac5f15e6cd8357ad0fc7ac66e3a2bf2c887
+size 33967272

zain/Activation/out/glu-linear-100L_run/checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e0935c4af80575686339f34649e089f4880c14898c3917ea0834e9ecbbfbacd
+size 68504996

zain/Activation/out/glu-linear-100L_run/checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-linear-100L_run/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ad40e275c0c46c262ecd103385af8fb68cf3f382102b4f63937b0dbd1c69cfc
+size 1064

zain/Activation/out/glu-linear-100L_run/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-500/trainer_state.json

@@ -0,0 +1,209 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.03370407819346141,
+  "eval_steps": 2498,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29943389892578,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3046875,
+      "learning_rate": 5.46e-05,
+      "loss": 8.036395263671874,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.834572601318359,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5718994140625,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.237230682373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1484375,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.864912414550782,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.509800720214844,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 1.0234375,
+      "learning_rate": 0.0002226,
+      "loss": 6.20410041809082,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.875,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 5.940516662597656,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.0002786,
+      "loss": 5.676180267333985,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.412886428833008,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.81640625,
+      "learning_rate": 0.0003346,
+      "loss": 5.135355377197266,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 4.882264709472656,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.0003906,
+      "loss": 4.6449634552001955,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0004186,
+      "loss": 4.446089553833008,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.0004466,
+      "loss": 4.2602802276611325,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.108924484252929,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.0005026,
+      "loss": 3.984059143066406,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.0005306,
+      "loss": 3.864672088623047,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.0005586,
+      "loss": 3.741139602661133,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.0005866,
+      "loss": 3.6620559692382812,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.578125,
+      "learning_rate": 0.0006146,
+      "loss": 3.604226303100586,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.484375,
+      "learning_rate": 0.0006426,
+      "loss": 3.5065567016601564,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.60546875,
+      "learning_rate": 0.0006705999999999999,
+      "loss": 3.437101364135742,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.57421875,
+      "learning_rate": 0.0006986,
+      "loss": 3.3709640502929688,
+      "step": 500
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1613141901312000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f7f80cd089898db6bc7e5512322b4c5393daccc1c74b5f3c88feb826b29d13
+size 4920

zain/Activation/out/glu-linear-100L_run/checkpoint-600/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "linear",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-linear-100L_run/checkpoint-600/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb837f1b9c19356c2862393eed63e01aa0eb5ee28dd3ec1c57bea2500ec4b172
+size 33967272

zain/Activation/out/glu-linear-100L_run/checkpoint-600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68335f4477a4320e2d545eec904d5534d166e5ba408611e5eabc208bc2af0583
+size 68504996

zain/Activation/out/glu-linear-100L_run/checkpoint-600/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-linear-100L_run/checkpoint-600/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43a21235ddd7c8c0020d887652dce105923f094e2bc54766bffa71522037a007
+size 1064