Auto upload zain 2026-08-12T23:10:46.993782

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "situglu_low",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fea21e6eba17a8e562dd3b2a464292b5336cb226d8cad7ce319792f640dca924
+size 33967272

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:342bfdb33bab95f50332a1e2d976a2b6224dc5b173677184caea7c33f701a0bd
+size 68504996

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3001486ba00eb51d89ef078fe70e7e37535cd4e2311d7dd589266d045c9fb892
+size 1064

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/trainer_state.json

@@ -0,0 +1,69 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.006740815638692282,
+  "eval_steps": 2498,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.299667358398438,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047861480712891,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.838032531738281,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.572964477539062,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.233274841308594,
+      "step": 100
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 322628380262400.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb8cca2a9ea6ff541b7562ec4e2e31a24b5072ef5136a6523ae9f9c0012caa0
+size 4920

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "situglu_low",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1022baab0bb6d75de76c7fadb03f1ded1ceacf6b19b6ae2f5bc125a3fc9e971
+size 33967272

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc873b34547c03df88563ccf0947991bba4cffeb90626c86a3b05cb4746920ef
+size 68504996

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28668eba09299968c820a8f82f4368de9f79bd22806c6be52cebe6b8700b6584
+size 1064

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/trainer_state.json

@@ -0,0 +1,104 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.013481631277384564,
+  "eval_steps": 2498,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.299667358398438,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047861480712891,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.838032531738281,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.572964477539062,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.233274841308594,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.860704803466797,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.517856597900391,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.0002226,
+      "loss": 6.239397811889648,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.65234375,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 6.039968109130859,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.5625,
+      "learning_rate": 0.0002786,
+      "loss": 5.863864517211914,
+      "step": 200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 645256760524800.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-situglu_low-100L_run/checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb8cca2a9ea6ff541b7562ec4e2e31a24b5072ef5136a6523ae9f9c0012caa0
+size 4920

zain/Activation/out/glu-situglu_low-100L_run/training_log.jsonl

@@ -0,0 +1,14 @@
+{"step": 20, "epoch": 0.0013481631277384564, "timestamp": 1786576162.6909053, "loss": 8.299667358398438, "grad_norm": 2.078125, "learning_rate": 2.66e-05, "train/total_time_seconds": 5.60199935361743, "train/time_per_step_avg": 0.2800999676808715, "train/epoch_time_elapsed": 6.6612160094082355, "train/estimated_remaining_minutes": 11.577465330809355}
+{"step": 40, "epoch": 0.002696326255476913, "timestamp": 1786576169.0410013, "loss": 8.047861480712891, "grad_norm": 1.3515625, "learning_rate": 5.46e-05, "train/total_time_seconds": 10.95127809792757, "train/time_per_step_avg": 0.27378195244818926, "train/epoch_time_elapsed": 13.011312425136566, "train/estimated_remaining_minutes": 11.22506005037576}
+{"step": 60, "epoch": 0.004044489383215369, "timestamp": 1786576175.3209577, "loss": 7.838032531738281, "grad_norm": 1.2734375, "learning_rate": 8.259999999999999e-05, "train/total_time_seconds": 16.258189603686333, "train/time_per_step_avg": 0.27096982672810555, "train/epoch_time_elapsed": 19.291268810629845, "train/estimated_remaining_minutes": 11.019439620276293}
+{"step": 80, "epoch": 0.005392652510953826, "timestamp": 1786576181.6440203, "loss": 7.572964477539062, "grad_norm": 1.265625, "learning_rate": 0.0001106, "train/total_time_seconds": 21.575381491333246, "train/time_per_step_avg": 0.26969226864166557, "train/epoch_time_elapsed": 25.614331372082233, "train/estimated_remaining_minutes": 10.87758816854718}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786576187.9613962, "loss": 7.233274841308594, "grad_norm": 1.1953125, "learning_rate": 0.0001386, "train/total_time_seconds": 26.904183961451054, "train/time_per_step_avg": 0.26904183961451056, "train/epoch_time_elapsed": 31.93170676007867, "train/estimated_remaining_minutes": 10.761673584580421}
+{"step": 120, "epoch": 0.008088978766430738, "timestamp": 1786576194.5339625, "loss": 6.860704803466797, "grad_norm": 1.1328125, "learning_rate": 0.00016659999999999998, "train/total_time_seconds": 32.21203510090709, "train/time_per_step_avg": 0.26610035747289656, "train/epoch_time_elapsed": 38.504273399710655, "train/estimated_remaining_minutes": 10.6478671583554}
+{"step": 140, "epoch": 0.009437141894169195, "timestamp": 1786576200.8192215, "loss": 6.517856597900391, "grad_norm": 0.94921875, "learning_rate": 0.00019460000000000001, "train/total_time_seconds": 37.514357071369886, "train/time_per_step_avg": 0.26563078973442317, "train/epoch_time_elapsed": 44.78953251987696, "train/estimated_remaining_minutes": 10.53974793909916}
+{"step": 160, "epoch": 0.010785305021907651, "timestamp": 1786576207.1259038, "loss": 6.239397811889648, "grad_norm": 0.8046875, "learning_rate": 0.0002226, "train/total_time_seconds": 42.8182353079319, "train/time_per_step_avg": 0.26560045704245566, "train/epoch_time_elapsed": 51.096214696764946, "train/estimated_remaining_minutes": 10.4369448563084}
+{"step": 180, "epoch": 0.012133468149646108, "timestamp": 1786576213.4923518, "loss": 6.039968109130859, "grad_norm": 0.65234375, "learning_rate": 0.00025059999999999997, "train/total_time_seconds": 48.18610752373934, "train/time_per_step_avg": 0.2661072603240609, "train/epoch_time_elapsed": 57.46266236901283, "train/estimated_remaining_minutes": 10.351089764358822}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786576220.059817, "loss": 5.863864517211914, "grad_norm": 0.5625, "learning_rate": 0.0002786, "train/total_time_seconds": 53.74020229652524, "train/time_per_step_avg": 0.26836018335074185, "train/epoch_time_elapsed": 64.03012705594301, "train/estimated_remaining_minutes": 10.300205440167337}
+{"step": 220, "epoch": 0.01482979440512302, "timestamp": 1786576226.6697376, "loss": 5.679952239990234, "grad_norm": 0.67578125, "learning_rate": 0.00030659999999999997, "train/total_time_seconds": 59.053341407328844, "train/time_per_step_avg": 0.2684130630642176, "train/epoch_time_elapsed": 70.64004838839173, "train/estimated_remaining_minutes": 10.200122606720436}
+{"step": 240, "epoch": 0.016177957532861477, "timestamp": 1786576232.9391959, "loss": 5.470568084716797, "grad_norm": 1.484375, "learning_rate": 0.0003346, "train/total_time_seconds": 64.34804996475577, "train/time_per_step_avg": 0.26833692893385885, "train/epoch_time_elapsed": 76.90950669720769, "train/estimated_remaining_minutes": 10.099068952801948}
+{"step": 260, "epoch": 0.01752612066059993, "timestamp": 1786576239.2550485, "loss": 5.266762542724609, "grad_norm": 1.9765625, "learning_rate": 0.00036260000000000003, "train/total_time_seconds": 69.67213932424784, "train/time_per_step_avg": 0.2685390401631594, "train/epoch_time_elapsed": 83.22535910457373, "train/estimated_remaining_minutes": 10.004204620917639}
+{"step": 280, "epoch": 0.01887428378833839, "timestamp": 1786576245.6118467, "loss": 5.04127197265625, "grad_norm": 0.6015625, "learning_rate": 0.0003906, "train/total_time_seconds": 75.00789070129395, "train/time_per_step_avg": 0.2682178317755461, "train/epoch_time_elapsed": 89.58215619623661, "train/estimated_remaining_minutes": 9.911756985528127}