Auto upload zain 2026-08-14T23:07:46.851605

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b810704666206911920bd86cac730f5e0e2b65890323c7298ebaa19dd00b1a30
+size 7961528

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8bfdf314d9e4b27ff75e5ae08adedd2131ac4625cc0ee847bdb664b29bea357
+size 16041274

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:906745ab5f61e73e8c1ba850ef3b839e2766eba889b49c843836266988895165
+size 1064

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/trainer_state.json

@@ -0,0 +1,69 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.05392289026691831,
+  "eval_steps": 60000,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.03125,
+      "learning_rate": 0.001,
+      "loss": 28.933071899414063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 1.328125,
+      "learning_rate": 0.001,
+      "loss": 24.235337829589845,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 2.84375,
+      "learning_rate": 0.001,
+      "loss": 22.72260284423828,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 21.375914001464842,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.125,
+      "learning_rate": 0.001,
+      "loss": 19.956423950195312,
+      "step": 100
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 542031583641600.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d1e8c6e2d9a481c9b0f55e934a1b3b8c583833193185af2788cf94ede357e2
+size 4920

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4a28362bb4c69b2256d8b11378c5538ba674c196b381757dcd5391ad5e42bf4
+size 7961528

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2e65cd818a729d7f589d2e2d02e866e93cc7f5892f6f453a1cc03b0ade1034
+size 16041274

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b66e3cc7c452b707ddac5caf0aa17618afb9bc1a0333600a22c4afb353f3165
+size 14244

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fc3ca8af69cc9d003a05a139e7997960ab393fe31ae1d898b29038573704b4e
+size 1064

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,392 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5392289026691831,
+  "eval_steps": 60000,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.03125,
+      "learning_rate": 0.001,
+      "loss": 28.933071899414063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 1.328125,
+      "learning_rate": 0.001,
+      "loss": 24.235337829589845,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 2.84375,
+      "learning_rate": 0.001,
+      "loss": 22.72260284423828,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 21.375914001464842,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.125,
+      "learning_rate": 0.001,
+      "loss": 19.956423950195312,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.078125,
+      "learning_rate": 0.001,
+      "loss": 18.432823181152344,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.65625,
+      "learning_rate": 0.001,
+      "loss": 17.21851043701172,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.001,
+      "loss": 16.266268920898437,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 15.486952209472657,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.796875,
+      "learning_rate": 0.001,
+      "loss": 14.80138702392578,
+      "step": 200
+    },
+    {
+      "epoch": 0.11863035858722028,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 14.216067504882812,
+      "step": 220
+    },
+    {
+      "epoch": 0.12941493664060394,
+      "grad_norm": 2.640625,
+      "learning_rate": 0.001,
+      "loss": 13.734385681152343,
+      "step": 240
+    },
+    {
+      "epoch": 0.1401995146939876,
+      "grad_norm": 2.171875,
+      "learning_rate": 0.001,
+      "loss": 13.254803466796876,
+      "step": 260
+    },
+    {
+      "epoch": 0.15098409274737126,
+      "grad_norm": 1.78125,
+      "learning_rate": 0.001,
+      "loss": 12.848719787597656,
+      "step": 280
+    },
+    {
+      "epoch": 0.1617686708007549,
+      "grad_norm": 2.5,
+      "learning_rate": 0.001,
+      "loss": 12.483526611328125,
+      "step": 300
+    },
+    {
+      "epoch": 0.17255324885413859,
+      "grad_norm": 3.0625,
+      "learning_rate": 0.001,
+      "loss": 12.143161010742187,
+      "step": 320
+    },
+    {
+      "epoch": 0.18333782690752223,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 11.856404876708984,
+      "step": 340
+    },
+    {
+      "epoch": 0.1941224049609059,
+      "grad_norm": 2.515625,
+      "learning_rate": 0.001,
+      "loss": 11.560836029052734,
+      "step": 360
+    },
+    {
+      "epoch": 0.20490698301428956,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.001,
+      "loss": 11.295862579345703,
+      "step": 380
+    },
+    {
+      "epoch": 0.21569156106767323,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.001,
+      "loss": 11.08508071899414,
+      "step": 400
+    },
+    {
+      "epoch": 0.22647613912105688,
+      "grad_norm": 3.015625,
+      "learning_rate": 0.001,
+      "loss": 10.860546875,
+      "step": 420
+    },
+    {
+      "epoch": 0.23726071717444056,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.001,
+      "loss": 10.721086883544922,
+      "step": 440
+    },
+    {
+      "epoch": 0.2480452952278242,
+      "grad_norm": 2.328125,
+      "learning_rate": 0.001,
+      "loss": 10.516852569580077,
+      "step": 460
+    },
+    {
+      "epoch": 0.2588298732812079,
+      "grad_norm": 2.9375,
+      "learning_rate": 0.001,
+      "loss": 10.34997329711914,
+      "step": 480
+    },
+    {
+      "epoch": 0.26961445133459155,
+      "grad_norm": 2.4375,
+      "learning_rate": 0.001,
+      "loss": 10.200981140136719,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803990293879752,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 10.089121246337891,
+      "step": 520
+    },
+    {
+      "epoch": 0.29118360744135885,
+      "grad_norm": 2.0,
+      "learning_rate": 0.001,
+      "loss": 9.964746856689453,
+      "step": 540
+    },
+    {
+      "epoch": 0.3019681854947425,
+      "grad_norm": 1.9921875,
+      "learning_rate": 0.001,
+      "loss": 9.853208923339844,
+      "step": 560
+    },
+    {
+      "epoch": 0.3127527635481262,
+      "grad_norm": 1.8515625,
+      "learning_rate": 0.001,
+      "loss": 9.7361083984375,
+      "step": 580
+    },
+    {
+      "epoch": 0.3235373416015098,
+      "grad_norm": 2.65625,
+      "learning_rate": 0.001,
+      "loss": 9.627728271484376,
+      "step": 600
+    },
+    {
+      "epoch": 0.3343219196548935,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 9.535695648193359,
+      "step": 620
+    },
+    {
+      "epoch": 0.34510649770827717,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 9.440611267089844,
+      "step": 640
+    },
+    {
+      "epoch": 0.35589107576166085,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.001,
+      "loss": 9.348371887207032,
+      "step": 660
+    },
+    {
+      "epoch": 0.36667565381504447,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.001,
+      "loss": 9.264537048339843,
+      "step": 680
+    },
+    {
+      "epoch": 0.37746023186842814,
+      "grad_norm": 1.640625,
+      "learning_rate": 0.001,
+      "loss": 9.190103149414062,
+      "step": 700
+    },
+    {
+      "epoch": 0.3882448099218118,
+      "grad_norm": 2.15625,
+      "learning_rate": 0.001,
+      "loss": 9.099945831298829,
+      "step": 720
+    },
+    {
+      "epoch": 0.3990293879751955,
+      "grad_norm": 2.53125,
+      "learning_rate": 0.001,
+      "loss": 9.054994201660156,
+      "step": 740
+    },
+    {
+      "epoch": 0.4098139660285791,
+      "grad_norm": 2.015625,
+      "learning_rate": 0.001,
+      "loss": 8.98128662109375,
+      "step": 760
+    },
+    {
+      "epoch": 0.4205985440819628,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 8.936003112792969,
+      "step": 780
+    },
+    {
+      "epoch": 0.43138312213534646,
+      "grad_norm": 2.15625,
+      "learning_rate": 0.001,
+      "loss": 8.83919906616211,
+      "step": 800
+    },
+    {
+      "epoch": 0.44216770018873014,
+      "grad_norm": 2.34375,
+      "learning_rate": 0.001,
+      "loss": 8.804035949707032,
+      "step": 820
+    },
+    {
+      "epoch": 0.45295227824211376,
+      "grad_norm": 2.28125,
+      "learning_rate": 0.001,
+      "loss": 8.767097473144531,
+      "step": 840
+    },
+    {
+      "epoch": 0.46373685629549743,
+      "grad_norm": 2.4375,
+      "learning_rate": 0.001,
+      "loss": 8.690902709960938,
+      "step": 860
+    },
+    {
+      "epoch": 0.4745214343488811,
+      "grad_norm": 1.828125,
+      "learning_rate": 0.001,
+      "loss": 8.652106475830077,
+      "step": 880
+    },
+    {
+      "epoch": 0.4853060124022648,
+      "grad_norm": 2.5,
+      "learning_rate": 0.001,
+      "loss": 8.595584106445312,
+      "step": 900
+    },
+    {
+      "epoch": 0.4960905904556484,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 8.547761535644531,
+      "step": 920
+    },
+    {
+      "epoch": 0.5068751685090321,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.001,
+      "loss": 8.526532745361328,
+      "step": 940
+    },
+    {
+      "epoch": 0.5176597465624158,
+      "grad_norm": 1.90625,
+      "learning_rate": 0.001,
+      "loss": 8.503388977050781,
+      "step": 960
+    },
+    {
+      "epoch": 0.5284443246157994,
+      "grad_norm": 2.140625,
+      "learning_rate": 0.001,
+      "loss": 8.466845703125,
+      "step": 980
+    },
+    {
+      "epoch": 0.5392289026691831,
+      "grad_norm": 2.0,
+      "learning_rate": 0.001,
+      "loss": 8.422817993164063,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5392289026691831,
+      "eval_loss": 2.101808547973633,
+      "eval_runtime": 9.4744,
+      "eval_samples_per_second": 1005.553,
+      "eval_steps_per_second": 7.916,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5420315836416000.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d1e8c6e2d9a481c9b0f55e934a1b3b8c583833193185af2788cf94ede357e2
+size 4920

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:971c4460dc541f66ff73f35dcf6303ad13a35b9d38aa22277787249ed8367ec2
+size 7961528

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa3854bafa0fc7ed438fcfe53c67e5ee9a1fda9ee927c0df413855481c53098d
+size 16041274

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3b5a970fc92af42664e84ba686059eaaacfaa668691c0cf6d2afddda677461
+size 1064

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/trainer_state.json

@@ -0,0 +1,104 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.10784578053383662,
+  "eval_steps": 60000,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.03125,
+      "learning_rate": 0.001,
+      "loss": 28.933071899414063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 1.328125,
+      "learning_rate": 0.001,
+      "loss": 24.235337829589845,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 2.84375,
+      "learning_rate": 0.001,
+      "loss": 22.72260284423828,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 21.375914001464842,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.125,
+      "learning_rate": 0.001,
+      "loss": 19.956423950195312,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.078125,
+      "learning_rate": 0.001,
+      "loss": 18.432823181152344,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.65625,
+      "learning_rate": 0.001,
+      "loss": 17.21851043701172,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.001,
+      "loss": 16.266268920898437,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 15.486952209472657,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.796875,
+      "learning_rate": 0.001,
+      "loss": 14.80138702392578,
+      "step": 200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1084063167283200.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d1e8c6e2d9a481c9b0f55e934a1b3b8c583833193185af2788cf94ede357e2
+size 4920

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65b978e34f8acce028c0c43b1326c9b9c4f4fbc81dcd6010edd4c6a834334b27
+size 7961528

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78a52a49491bfef478454781044f90ae0974b5d4eb7ebe7b7914f9a9f0b673c6
+size 16041274

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76cd21f32221b90fb8288fd2428595ec8b5039fe2425086d67cacb5c8d814c9e
+size 1064

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/trainer_state.json

@@ -0,0 +1,139 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1617686708007549,
+  "eval_steps": 60000,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.03125,
+      "learning_rate": 0.001,
+      "loss": 28.933071899414063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 1.328125,
+      "learning_rate": 0.001,
+      "loss": 24.235337829589845,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 2.84375,
+      "learning_rate": 0.001,
+      "loss": 22.72260284423828,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 21.375914001464842,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.125,
+      "learning_rate": 0.001,
+      "loss": 19.956423950195312,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.078125,
+      "learning_rate": 0.001,
+      "loss": 18.432823181152344,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.65625,
+      "learning_rate": 0.001,
+      "loss": 17.21851043701172,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.001,
+      "loss": 16.266268920898437,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 15.486952209472657,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.796875,
+      "learning_rate": 0.001,
+      "loss": 14.80138702392578,
+      "step": 200
+    },
+    {
+      "epoch": 0.11863035858722028,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 14.216067504882812,
+      "step": 220
+    },
+    {
+      "epoch": 0.12941493664060394,
+      "grad_norm": 2.640625,
+      "learning_rate": 0.001,
+      "loss": 13.734385681152343,
+      "step": 240
+    },
+    {
+      "epoch": 0.1401995146939876,
+      "grad_norm": 2.171875,
+      "learning_rate": 0.001,
+      "loss": 13.254803466796876,
+      "step": 260
+    },
+    {
+      "epoch": 0.15098409274737126,
+      "grad_norm": 1.78125,
+      "learning_rate": 0.001,
+      "loss": 12.848719787597656,
+      "step": 280
+    },
+    {
+      "epoch": 0.1617686708007549,
+      "grad_norm": 2.5,
+      "learning_rate": 0.001,
+      "loss": 12.483526611328125,
+      "step": 300
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1626094750924800.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d1e8c6e2d9a481c9b0f55e934a1b3b8c583833193185af2788cf94ede357e2
+size 4920

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15b6447f577e82b970ad1b9fa39c7f2a789ca87590f25fe511c3e5f9cd787e31
+size 7961528

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32e38bf4181ddc915a97030e5d6840174987770d4b3bd550064fe3d446bbe872
+size 16041274

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51769a11c87b2b665cbe64c58b934afdb1fa1998bffb4addcc2f852b172d6681
+size 1064

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/trainer_state.json

@@ -0,0 +1,174 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.21569156106767323,
+  "eval_steps": 60000,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.03125,
+      "learning_rate": 0.001,
+      "loss": 28.933071899414063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 1.328125,
+      "learning_rate": 0.001,
+      "loss": 24.235337829589845,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 2.84375,
+      "learning_rate": 0.001,
+      "loss": 22.72260284423828,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 21.375914001464842,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.125,
+      "learning_rate": 0.001,
+      "loss": 19.956423950195312,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.078125,
+      "learning_rate": 0.001,
+      "loss": 18.432823181152344,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.65625,
+      "learning_rate": 0.001,
+      "loss": 17.21851043701172,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.001,
+      "loss": 16.266268920898437,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 15.486952209472657,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.796875,
+      "learning_rate": 0.001,
+      "loss": 14.80138702392578,
+      "step": 200
+    },
+    {
+      "epoch": 0.11863035858722028,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 14.216067504882812,
+      "step": 220
+    },
+    {
+      "epoch": 0.12941493664060394,
+      "grad_norm": 2.640625,
+      "learning_rate": 0.001,
+      "loss": 13.734385681152343,
+      "step": 240
+    },
+    {
+      "epoch": 0.1401995146939876,
+      "grad_norm": 2.171875,
+      "learning_rate": 0.001,
+      "loss": 13.254803466796876,
+      "step": 260
+    },
+    {
+      "epoch": 0.15098409274737126,
+      "grad_norm": 1.78125,
+      "learning_rate": 0.001,
+      "loss": 12.848719787597656,
+      "step": 280
+    },
+    {
+      "epoch": 0.1617686708007549,
+      "grad_norm": 2.5,
+      "learning_rate": 0.001,
+      "loss": 12.483526611328125,
+      "step": 300
+    },
+    {
+      "epoch": 0.17255324885413859,
+      "grad_norm": 3.0625,
+      "learning_rate": 0.001,
+      "loss": 12.143161010742187,
+      "step": 320
+    },
+    {
+      "epoch": 0.18333782690752223,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 11.856404876708984,
+      "step": 340
+    },
+    {
+      "epoch": 0.1941224049609059,
+      "grad_norm": 2.515625,
+      "learning_rate": 0.001,
+      "loss": 11.560836029052734,
+      "step": 360
+    },
+    {
+      "epoch": 0.20490698301428956,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.001,
+      "loss": 11.295862579345703,
+      "step": 380
+    },
+    {
+      "epoch": 0.21569156106767323,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.001,
+      "loss": 11.08508071899414,
+      "step": 400
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2168126334566400.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d1e8c6e2d9a481c9b0f55e934a1b3b8c583833193185af2788cf94ede357e2
+size 4920

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-500/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a3b7e07ab9c6ecdbd8218e569f1a30628ae62f6e57e651013394a6ee84a6349
+size 7961528

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8838775212f6530687a9118723add11d3ab3e6ca511fbe6ab8937ef15fc7d91f
+size 16041274

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/glu-silu-waleed10-21L_run/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60a0752f1683b6bad1b285850c7a77f8c14fbcad243554c3a99f6e443a78365e
+size 1064