Auto upload zain 2026-08-12T21:24:09.274426

zain/Activation/out/glu-situglu_low-150L_run/training_log.jsonl

@@ -0,0 +1,2 @@
+{"step": 20, "epoch": 0.0006740815638692282, "timestamp": 1786569824.8138995, "loss": 8.23435287475586, "grad_norm": 1.4453125, "learning_rate": 6.65e-05, "train/total_time_seconds": 7.173142466694117, "train/time_per_step_avg": 0.35865712333470584, "train/epoch_time_elapsed": 7.740738794207573, "train/estimated_remaining_minutes": 2.8692569866776467}
+{"step": 40, "epoch": 0.0013481631277384564, "timestamp": 1786569832.2752674, "loss": 7.900041961669922, "grad_norm": 1.375, "learning_rate": 0.0001365, "train/total_time_seconds": 14.130188584327698, "train/time_per_step_avg": 0.35325471460819247, "train/epoch_time_elapsed": 15.202106323093176, "train/estimated_remaining_minutes": 2.7082861453294753}

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:498fd09549d49fd93deb0f300fc27f403b2b9a41bcafed1b8ae7404b33ee4371
+size 50427120

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61f681059f6321e75996f6c19bf7e545c579c01845296d8cc718d9f57c604dd9
+size 101708778

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecefbb3f17bb76b6655eb0157c98b5287c17fa4b4c72a6b9068b0823ce9fd18d
+size 14244

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65adfad786ae2b224643009c3ac2998e8fc0229311e4b753becffd2fb9569fe9
+size 1064

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/trainer_state.json

@@ -0,0 +1,136 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.006740815638692282,
+  "eval_steps": 50,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0006740815638692282,
+      "grad_norm": 1.3203125,
+      "learning_rate": 6.65e-05,
+      "loss": 8.226606750488282,
+      "step": 20
+    },
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.0001365,
+      "loss": 7.892790985107422,
+      "step": 40
+    },
+    {
+      "epoch": 0.0016852039096730705,
+      "eval_loss": 7.459151268005371,
+      "eval_runtime": 20.6744,
+      "eval_samples_per_second": 460.811,
+      "eval_steps_per_second": 0.484,
+      "step": 50
+    },
+    {
+      "epoch": 0.0020222446916076846,
+      "grad_norm": 1.234375,
+      "learning_rate": 0.00020649999999999998,
+      "loss": 7.4655517578125,
+      "step": 60
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0002765,
+      "loss": 6.900899505615234,
+      "step": 80
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.0003465,
+      "loss": 6.384212875366211,
+      "step": 100
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.164856433868408,
+      "eval_runtime": 20.6327,
+      "eval_samples_per_second": 461.743,
+      "eval_steps_per_second": 0.485,
+      "step": 100
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.0004165,
+      "loss": 6.051737594604492,
+      "step": 120
+    },
+    {
+      "epoch": 0.0047185709470845974,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.00048649999999999995,
+      "loss": 5.780973052978515,
+      "step": 140
+    },
+    {
+      "epoch": 0.005055611729019211,
+      "eval_loss": 5.543824195861816,
+      "eval_runtime": 20.6322,
+      "eval_samples_per_second": 461.755,
+      "eval_steps_per_second": 0.485,
+      "step": 150
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0005565,
+      "loss": 5.545684051513672,
+      "step": 160
+    },
+    {
+      "epoch": 0.006066734074823054,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006265,
+      "loss": 5.289410018920899,
+      "step": 180
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.5390625,
+      "learning_rate": 0.0006965,
+      "loss": 4.994114685058594,
+      "step": 200
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 4.839845180511475,
+      "eval_runtime": 20.6673,
+      "eval_samples_per_second": 460.971,
+      "eval_steps_per_second": 0.484,
+      "step": 200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 483941312102400.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b827f60f983e3f5ad52047a134946ee6cd902cd1de8cdbd39e8e9aa340a1aef1
+size 4920

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ed98e05fae7de919d1730d84d7b1acfa96565be7a3dd2368178203c47718d5
+size 50427120

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:600ab3f5dfd1a0ec0bdf1c9e4dac80193572d54ead651deb0c54f9b4ae16e591
+size 101708778

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:068fbd993087219c15b8c0baa13fc39644a4dcdfe92d8be3fa6434deece90371
+size 14244

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7275519db1cfbc1c3b3ff4cc789ae435d90674550d32700685019af96bdd8863
+size 1064

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/trainer_state.json

@@ -0,0 +1,187 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.010111223458038422,
+  "eval_steps": 50,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0006740815638692282,
+      "grad_norm": 1.3203125,
+      "learning_rate": 6.65e-05,
+      "loss": 8.226606750488282,
+      "step": 20
+    },
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.0001365,
+      "loss": 7.892790985107422,
+      "step": 40
+    },
+    {
+      "epoch": 0.0016852039096730705,
+      "eval_loss": 7.459151268005371,
+      "eval_runtime": 20.6744,
+      "eval_samples_per_second": 460.811,
+      "eval_steps_per_second": 0.484,
+      "step": 50
+    },
+    {
+      "epoch": 0.0020222446916076846,
+      "grad_norm": 1.234375,
+      "learning_rate": 0.00020649999999999998,
+      "loss": 7.4655517578125,
+      "step": 60
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0002765,
+      "loss": 6.900899505615234,
+      "step": 80
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.0003465,
+      "loss": 6.384212875366211,
+      "step": 100
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.164856433868408,
+      "eval_runtime": 20.6327,
+      "eval_samples_per_second": 461.743,
+      "eval_steps_per_second": 0.485,
+      "step": 100
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.0004165,
+      "loss": 6.051737594604492,
+      "step": 120
+    },
+    {
+      "epoch": 0.0047185709470845974,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.00048649999999999995,
+      "loss": 5.780973052978515,
+      "step": 140
+    },
+    {
+      "epoch": 0.005055611729019211,
+      "eval_loss": 5.543824195861816,
+      "eval_runtime": 20.6322,
+      "eval_samples_per_second": 461.755,
+      "eval_steps_per_second": 0.485,
+      "step": 150
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0005565,
+      "loss": 5.545684051513672,
+      "step": 160
+    },
+    {
+      "epoch": 0.006066734074823054,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006265,
+      "loss": 5.289410018920899,
+      "step": 180
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.5390625,
+      "learning_rate": 0.0006965,
+      "loss": 4.994114685058594,
+      "step": 200
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 4.839845180511475,
+      "eval_runtime": 20.6673,
+      "eval_samples_per_second": 460.971,
+      "eval_steps_per_second": 0.484,
+      "step": 200
+    },
+    {
+      "epoch": 0.00741489720256151,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.0007,
+      "loss": 4.748640823364258,
+      "step": 220
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.0007,
+      "loss": 4.502994537353516,
+      "step": 240
+    },
+    {
+      "epoch": 0.008426019548365353,
+      "eval_loss": 4.319539546966553,
+      "eval_runtime": 20.6786,
+      "eval_samples_per_second": 460.717,
+      "eval_steps_per_second": 0.484,
+      "step": 250
+    },
+    {
+      "epoch": 0.008763060330299966,
+      "grad_norm": 0.455078125,
+      "learning_rate": 0.0007,
+      "loss": 4.330507278442383,
+      "step": 260
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.0007,
+      "loss": 4.185513687133789,
+      "step": 280
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.0007,
+      "loss": 4.058372116088867,
+      "step": 300
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 4.001541614532471,
+      "eval_runtime": 20.746,
+      "eval_samples_per_second": 459.22,
+      "eval_steps_per_second": 0.482,
+      "step": 300
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 725911968153600.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b827f60f983e3f5ad52047a134946ee6cd902cd1de8cdbd39e8e9aa340a1aef1
+size 4920

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acf1ce7f138d292235a0cd10eafffd59a9aa9ca60d99858356f0394236830d2b
+size 50427120

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7fe06aeac9ea87d94f255285ddbece675611441794defb92668c91d36e1c5d
+size 101708778

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9a6944c405a002fce05f295d08ea6650e2e2ad6dbf5d6da1e9053f7bf7f5827
+size 14244

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59985ace3cc91dc071d22964a9d7ec324bcb84a61c5666f3264564df2f5797e4
+size 1064

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/trainer_state.json

@@ -0,0 +1,238 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.013481631277384564,
+  "eval_steps": 50,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0006740815638692282,
+      "grad_norm": 1.3203125,
+      "learning_rate": 6.65e-05,
+      "loss": 8.226606750488282,
+      "step": 20
+    },
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.0001365,
+      "loss": 7.892790985107422,
+      "step": 40
+    },
+    {
+      "epoch": 0.0016852039096730705,
+      "eval_loss": 7.459151268005371,
+      "eval_runtime": 20.6744,
+      "eval_samples_per_second": 460.811,
+      "eval_steps_per_second": 0.484,
+      "step": 50
+    },
+    {
+      "epoch": 0.0020222446916076846,
+      "grad_norm": 1.234375,
+      "learning_rate": 0.00020649999999999998,
+      "loss": 7.4655517578125,
+      "step": 60
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0002765,
+      "loss": 6.900899505615234,
+      "step": 80
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.0003465,
+      "loss": 6.384212875366211,
+      "step": 100
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.164856433868408,
+      "eval_runtime": 20.6327,
+      "eval_samples_per_second": 461.743,
+      "eval_steps_per_second": 0.485,
+      "step": 100
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.0004165,
+      "loss": 6.051737594604492,
+      "step": 120
+    },
+    {
+      "epoch": 0.0047185709470845974,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.00048649999999999995,
+      "loss": 5.780973052978515,
+      "step": 140
+    },
+    {
+      "epoch": 0.005055611729019211,
+      "eval_loss": 5.543824195861816,
+      "eval_runtime": 20.6322,
+      "eval_samples_per_second": 461.755,
+      "eval_steps_per_second": 0.485,
+      "step": 150
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0005565,
+      "loss": 5.545684051513672,
+      "step": 160
+    },
+    {
+      "epoch": 0.006066734074823054,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006265,
+      "loss": 5.289410018920899,
+      "step": 180
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.5390625,
+      "learning_rate": 0.0006965,
+      "loss": 4.994114685058594,
+      "step": 200
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 4.839845180511475,
+      "eval_runtime": 20.6673,
+      "eval_samples_per_second": 460.971,
+      "eval_steps_per_second": 0.484,
+      "step": 200
+    },
+    {
+      "epoch": 0.00741489720256151,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.0007,
+      "loss": 4.748640823364258,
+      "step": 220
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.0007,
+      "loss": 4.502994537353516,
+      "step": 240
+    },
+    {
+      "epoch": 0.008426019548365353,
+      "eval_loss": 4.319539546966553,
+      "eval_runtime": 20.6786,
+      "eval_samples_per_second": 460.717,
+      "eval_steps_per_second": 0.484,
+      "step": 250
+    },
+    {
+      "epoch": 0.008763060330299966,
+      "grad_norm": 0.455078125,
+      "learning_rate": 0.0007,
+      "loss": 4.330507278442383,
+      "step": 260
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.0007,
+      "loss": 4.185513687133789,
+      "step": 280
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.0007,
+      "loss": 4.058372116088867,
+      "step": 300
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 4.001541614532471,
+      "eval_runtime": 20.746,
+      "eval_samples_per_second": 459.22,
+      "eval_steps_per_second": 0.482,
+      "step": 300
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.0007,
+      "loss": 3.9491451263427733,
+      "step": 320
+    },
+    {
+      "epoch": 0.011459386585776879,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.0007,
+      "loss": 3.8855216979980467,
+      "step": 340
+    },
+    {
+      "epoch": 0.011796427367711493,
+      "eval_loss": 3.792919874191284,
+      "eval_runtime": 20.7384,
+      "eval_samples_per_second": 459.39,
+      "eval_steps_per_second": 0.482,
+      "step": 350
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.78214111328125,
+      "step": 360
+    },
+    {
+      "epoch": 0.012807549713515335,
+      "grad_norm": 0.478515625,
+      "learning_rate": 0.0007,
+      "loss": 3.7168258666992187,
+      "step": 380
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.660007858276367,
+      "step": 400
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.6307969093322754,
+      "eval_runtime": 20.7141,
+      "eval_samples_per_second": 459.927,
+      "eval_steps_per_second": 0.483,
+      "step": 400
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 967882624204800.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b827f60f983e3f5ad52047a134946ee6cd902cd1de8cdbd39e8e9aa340a1aef1
+size 4920

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee8cdea3b2881be1f0bf35b491fcbc952c771b9e744b1d72e647cde565cb331a
+size 50427120

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a16e55a641d7ec7ed033f9c9abce7806443b9c4e9a5c6a80989dec3c2bfa07ee
+size 101708778

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a97db8e41139aa1239ba7fb79ddeb0af5998c6305a440c1fe182e6ad02f2f5
+size 14244

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ad40e275c0c46c262ecd103385af8fb68cf3f382102b4f63937b0dbd1c69cfc
+size 1064

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/trainer_state.json

@@ -0,0 +1,289 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.016852039096730706,
+  "eval_steps": 50,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0006740815638692282,
+      "grad_norm": 1.3203125,
+      "learning_rate": 6.65e-05,
+      "loss": 8.226606750488282,
+      "step": 20
+    },
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.0001365,
+      "loss": 7.892790985107422,
+      "step": 40
+    },
+    {
+      "epoch": 0.0016852039096730705,
+      "eval_loss": 7.459151268005371,
+      "eval_runtime": 20.6744,
+      "eval_samples_per_second": 460.811,
+      "eval_steps_per_second": 0.484,
+      "step": 50
+    },
+    {
+      "epoch": 0.0020222446916076846,
+      "grad_norm": 1.234375,
+      "learning_rate": 0.00020649999999999998,
+      "loss": 7.4655517578125,
+      "step": 60
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0002765,
+      "loss": 6.900899505615234,
+      "step": 80
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.0003465,
+      "loss": 6.384212875366211,
+      "step": 100
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.164856433868408,
+      "eval_runtime": 20.6327,
+      "eval_samples_per_second": 461.743,
+      "eval_steps_per_second": 0.485,
+      "step": 100
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.0004165,
+      "loss": 6.051737594604492,
+      "step": 120
+    },
+    {
+      "epoch": 0.0047185709470845974,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.00048649999999999995,
+      "loss": 5.780973052978515,
+      "step": 140
+    },
+    {
+      "epoch": 0.005055611729019211,
+      "eval_loss": 5.543824195861816,
+      "eval_runtime": 20.6322,
+      "eval_samples_per_second": 461.755,
+      "eval_steps_per_second": 0.485,
+      "step": 150
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0005565,
+      "loss": 5.545684051513672,
+      "step": 160
+    },
+    {
+      "epoch": 0.006066734074823054,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006265,
+      "loss": 5.289410018920899,
+      "step": 180
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.5390625,
+      "learning_rate": 0.0006965,
+      "loss": 4.994114685058594,
+      "step": 200
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 4.839845180511475,
+      "eval_runtime": 20.6673,
+      "eval_samples_per_second": 460.971,
+      "eval_steps_per_second": 0.484,
+      "step": 200
+    },
+    {
+      "epoch": 0.00741489720256151,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.0007,
+      "loss": 4.748640823364258,
+      "step": 220
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.0007,
+      "loss": 4.502994537353516,
+      "step": 240
+    },
+    {
+      "epoch": 0.008426019548365353,
+      "eval_loss": 4.319539546966553,
+      "eval_runtime": 20.6786,
+      "eval_samples_per_second": 460.717,
+      "eval_steps_per_second": 0.484,
+      "step": 250
+    },
+    {
+      "epoch": 0.008763060330299966,
+      "grad_norm": 0.455078125,
+      "learning_rate": 0.0007,
+      "loss": 4.330507278442383,
+      "step": 260
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.0007,
+      "loss": 4.185513687133789,
+      "step": 280
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.0007,
+      "loss": 4.058372116088867,
+      "step": 300
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 4.001541614532471,
+      "eval_runtime": 20.746,
+      "eval_samples_per_second": 459.22,
+      "eval_steps_per_second": 0.482,
+      "step": 300
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.0007,
+      "loss": 3.9491451263427733,
+      "step": 320
+    },
+    {
+      "epoch": 0.011459386585776879,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.0007,
+      "loss": 3.8855216979980467,
+      "step": 340
+    },
+    {
+      "epoch": 0.011796427367711493,
+      "eval_loss": 3.792919874191284,
+      "eval_runtime": 20.7384,
+      "eval_samples_per_second": 459.39,
+      "eval_steps_per_second": 0.482,
+      "step": 350
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.78214111328125,
+      "step": 360
+    },
+    {
+      "epoch": 0.012807549713515335,
+      "grad_norm": 0.478515625,
+      "learning_rate": 0.0007,
+      "loss": 3.7168258666992187,
+      "step": 380
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.660007858276367,
+      "step": 400
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.6307969093322754,
+      "eval_runtime": 20.7141,
+      "eval_samples_per_second": 459.927,
+      "eval_steps_per_second": 0.483,
+      "step": 400
+    },
+    {
+      "epoch": 0.014155712841253791,
+      "grad_norm": 0.55078125,
+      "learning_rate": 0.0007,
+      "loss": 3.5877830505371096,
+      "step": 420
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0007,
+      "loss": 3.560054397583008,
+      "step": 440
+    },
+    {
+      "epoch": 0.015166835187057633,
+      "eval_loss": 3.5101380348205566,
+      "eval_runtime": 20.5828,
+      "eval_samples_per_second": 462.863,
+      "eval_steps_per_second": 0.486,
+      "step": 450
+    },
+    {
+      "epoch": 0.015503875968992248,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.0007,
+      "loss": 3.5127296447753906,
+      "step": 460
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.43359375,
+      "learning_rate": 0.0007,
+      "loss": 3.4453048706054688,
+      "step": 480
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.431640625,
+      "learning_rate": 0.0007,
+      "loss": 3.415237045288086,
+      "step": 500
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 3.402008295059204,
+      "eval_runtime": 20.7141,
+      "eval_samples_per_second": 459.929,
+      "eval_steps_per_second": 0.483,
+      "step": 500
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1209853280256000.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-waleed-150L_run/checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b827f60f983e3f5ad52047a134946ee6cd902cd1de8cdbd39e8e9aa340a1aef1
+size 4920

zain/Activation/out/glu-waleed-150L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-waleed-150L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee8cdea3b2881be1f0bf35b491fcbc952c771b9e744b1d72e647cde565cb331a
+size 50427120

zain/Activation/out/glu-waleed-150L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-waleed-150L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b827f60f983e3f5ad52047a134946ee6cd902cd1de8cdbd39e8e9aa340a1aef1
+size 4920

zain/Activation/out/glu-waleed-150L_run/training_log.jsonl

@@ -11,3 +11,27 @@
 {"step": 160, "epoch": 0.005392652510953826, "timestamp": 1786569522.1553469, "loss": 5.545684051513672, "grad_norm": 0.98046875, "learning_rate": 0.0005565, "train/total_time_seconds": 55.305282179266214, "train/time_per_step_avg": 0.34286197159439324, "train/epoch_time_elapsed": 122.13275729864836, "train/estimated_remaining_minutes": 1.9587287438490115}
 {"step": 180, "epoch": 0.006066734074823054, "timestamp": 1786569529.4016442, "loss": 5.289410018920899, "grad_norm": 0.62109375, "learning_rate": 0.0006265, "train/total_time_seconds": 62.03357548639178, "train/time_per_step_avg": 0.3401971862465143, "train/epoch_time_elapsed": 129.37905456498265, "train/estimated_remaining_minutes": 1.8380318662634603}
 {"step": 200, "epoch": 0.006740815638692282, "timestamp": 1786569536.7232873, "loss": 4.994114685058594, "grad_norm": 0.5390625, "learning_rate": 0.0006965, "train/total_time_seconds": 68.81566143780947, "train/time_per_step_avg": 0.33934433944523335, "train/epoch_time_elapsed": 136.7006975747645, "train/estimated_remaining_minutes": 1.7203915359452369}
+{"step": 200, "epoch": 0.006740815638692282, "timestamp": 1786569557.3919575, "eval_loss": 4.839845180511475, "eval_runtime": 20.6673, "eval_samples_per_second": 460.971, "eval_steps_per_second": 0.484, "train/total_time_seconds": 68.81566143780947, "train/time_per_step_avg": 0.33934433944523335, "train/epoch_time_elapsed": 157.3693672977388, "train/estimated_remaining_minutes": 1.7203915359452369}
+{"step": 220, "epoch": 0.00741489720256151, "timestamp": 1786569565.3659294, "loss": 4.748640823364258, "grad_norm": 1.0703125, "learning_rate": 0.0007, "train/total_time_seconds": 75.62983629480004, "train/time_per_step_avg": 0.3397757708281279, "train/epoch_time_elapsed": 165.343339856714, "train/estimated_remaining_minutes": 1.6042692547381827}
+{"step": 240, "epoch": 0.008088978766430738, "timestamp": 1786569572.7260487, "loss": 4.502994537353516, "grad_norm": 0.6796875, "learning_rate": 0.0007, "train/total_time_seconds": 82.4601291641593, "train/time_per_step_avg": 0.34021866325289013, "train/epoch_time_elapsed": 172.70345924049616, "train/estimated_remaining_minutes": 1.4888634432417651}
+{"step": 250, "epoch": 0.008426019548365353, "timestamp": 1786569597.0672214, "eval_loss": 4.319539546966553, "eval_runtime": 20.6786, "eval_samples_per_second": 460.717, "eval_steps_per_second": 0.484, "train/total_time_seconds": 85.84956888481975, "train/time_per_step_avg": 0.3395453579351306, "train/epoch_time_elapsed": 197.04463040456176, "train/estimated_remaining_minutes": 1.430826148080329}
+{"step": 260, "epoch": 0.008763060330299966, "timestamp": 1786569600.7534935, "loss": 4.330507278442383, "grad_norm": 0.455078125, "learning_rate": 0.0007, "train/total_time_seconds": 89.27321784198284, "train/time_per_step_avg": 0.33967935662716625, "train/epoch_time_elapsed": 200.73090418428183, "train/estimated_remaining_minutes": 1.3734341206458898}
+{"step": 280, "epoch": 0.009437141894169195, "timestamp": 1786569608.106018, "loss": 4.185513687133789, "grad_norm": 0.59765625, "learning_rate": 0.0007, "train/total_time_seconds": 96.10496650263667, "train/time_per_step_avg": 0.3407139101624489, "train/epoch_time_elapsed": 208.0834284685552, "train/estimated_remaining_minutes": 1.2585174184869088}
+{"step": 300, "epoch": 0.010111223458038422, "timestamp": 1786569615.3244562, "loss": 4.058372116088867, "grad_norm": 0.62890625, "learning_rate": 0.0007, "train/total_time_seconds": 102.80648648738861, "train/time_per_step_avg": 0.33990825049579143, "train/epoch_time_elapsed": 215.30186684429646, "train/estimated_remaining_minutes": 1.142294294304318}
+{"step": 300, "epoch": 0.010111223458038422, "timestamp": 1786569636.0719028, "eval_loss": 4.001541614532471, "eval_runtime": 20.746, "eval_samples_per_second": 459.22, "eval_steps_per_second": 0.482, "train/total_time_seconds": 102.80648648738861, "train/time_per_step_avg": 0.33990825049579143, "train/epoch_time_elapsed": 236.0493126027286, "train/estimated_remaining_minutes": 1.142294294304318}
+{"step": 320, "epoch": 0.010785305021907651, "timestamp": 1786569644.0682247, "loss": 3.9491451263427733, "grad_norm": 0.54296875, "learning_rate": 0.0007, "train/total_time_seconds": 109.62384182959795, "train/time_per_step_avg": 0.33994005534797905, "train/epoch_time_elapsed": 244.04563404619694, "train/estimated_remaining_minutes": 1.0277235171524808}
+{"step": 340, "epoch": 0.011459386585776879, "timestamp": 1786569651.4826317, "loss": 3.8855216979980467, "grad_norm": 0.69921875, "learning_rate": 0.0007, "train/total_time_seconds": 116.5046098344028, "train/time_per_step_avg": 0.34044480670243504, "train/epoch_time_elapsed": 251.46004226058722, "train/estimated_remaining_minutes": 0.913761645760022}
+{"step": 350, "epoch": 0.011796427367711493, "timestamp": 1786569675.8987064, "eval_loss": 3.792919874191284, "eval_runtime": 20.7384, "eval_samples_per_second": 459.39, "eval_steps_per_second": 0.482, "train/total_time_seconds": 119.91959270089865, "train/time_per_step_avg": 0.34070023816078904, "train/epoch_time_elapsed": 275.87611592933536, "train/estimated_remaining_minutes": 0.8565685192921333}
+{"step": 360, "epoch": 0.012133468149646108, "timestamp": 1786569679.5336637, "loss": 3.78214111328125, "grad_norm": 0.53515625, "learning_rate": 0.0007, "train/total_time_seconds": 123.29239882901311, "train/time_per_step_avg": 0.3401918098703027, "train/epoch_time_elapsed": 279.5110743716359, "train/estimated_remaining_minutes": 0.7991173998176776}
+{"step": 380, "epoch": 0.012807549713515335, "timestamp": 1786569686.8399236, "loss": 3.7168258666992187, "grad_norm": 0.478515625, "learning_rate": 0.0007, "train/total_time_seconds": 130.06856789067388, "train/time_per_step_avg": 0.33963601388037207, "train/epoch_time_elapsed": 286.8173341713846, "train/estimated_remaining_minutes": 0.6845714099509151}
+{"step": 400, "epoch": 0.013481631277384564, "timestamp": 1786569694.158065, "loss": 3.660007858276367, "grad_norm": 0.48046875, "learning_rate": 0.0007, "train/total_time_seconds": 136.86465257406235, "train/time_per_step_avg": 0.34058166086673736, "train/epoch_time_elapsed": 294.1354754194617, "train/estimated_remaining_minutes": 0.5702693857252598}
+{"step": 400, "epoch": 0.013481631277384564, "timestamp": 1786569714.873715, "eval_loss": 3.6307969093322754, "eval_runtime": 20.7141, "eval_samples_per_second": 459.927, "eval_steps_per_second": 0.483, "train/total_time_seconds": 136.86465257406235, "train/time_per_step_avg": 0.34058166086673736, "train/epoch_time_elapsed": 314.8511252440512, "train/estimated_remaining_minutes": 0.5702693857252598}
+{"step": 420, "epoch": 0.014155712841253791, "timestamp": 1786569722.7422597, "loss": 3.5877830505371096, "grad_norm": 0.55078125, "learning_rate": 0.0007, "train/total_time_seconds": 143.5808736383915, "train/time_per_step_avg": 0.3395703180879355, "train/epoch_time_elapsed": 322.7196694277227, "train/estimated_remaining_minutes": 0.4558122972647349}
+{"step": 440, "epoch": 0.01482979440512302, "timestamp": 1786569730.1023037, "loss": 3.560054397583008, "grad_norm": 0.51953125, "learning_rate": 0.0007, "train/total_time_seconds": 150.4077126979828, "train/time_per_step_avg": 0.33903102863579987, "train/epoch_time_elapsed": 330.07971423864365, "train/estimated_remaining_minutes": 0.34183571067723356}
+{"step": 450, "epoch": 0.015166835187057633, "timestamp": 1786569754.3255029, "eval_loss": 3.5101380348205566, "eval_runtime": 20.5828, "eval_samples_per_second": 462.863, "eval_steps_per_second": 0.486, "train/total_time_seconds": 153.7891417145729, "train/time_per_step_avg": 0.3386954901367426, "train/epoch_time_elapsed": 354.30291237682104, "train/estimated_remaining_minutes": 0.28479470687883873}
+{"step": 460, "epoch": 0.015503875968992248, "timestamp": 1786569758.0804605, "loss": 3.5127296447753906, "grad_norm": 0.52734375, "learning_rate": 0.0007, "train/total_time_seconds": 157.28131246194243, "train/time_per_step_avg": 0.33988913632929324, "train/epoch_time_elapsed": 358.0578713901341, "train/estimated_remaining_minutes": 0.22794393110426442}
+{"step": 480, "epoch": 0.016177957532861477, "timestamp": 1786569765.378189, "loss": 3.4453048706054688, "grad_norm": 0.43359375, "learning_rate": 0.0007, "train/total_time_seconds": 164.06114525720477, "train/time_per_step_avg": 0.33992577366530896, "train/epoch_time_elapsed": 365.35559993982315, "train/estimated_remaining_minutes": 0.11393135087305888}
+{"step": 500, "epoch": 0.016852039096730706, "timestamp": 1786569772.603253, "loss": 3.415237045288086, "grad_norm": 0.431640625, "learning_rate": 0.0007, "train/total_time_seconds": 170.76810985431075, "train/time_per_step_avg": 0.33903457280248406, "train/epoch_time_elapsed": 372.5806630514562, "train/estimated_remaining_minutes": 0.0}
+{"step": 500, "epoch": 0.016852039096730706, "timestamp": 1786569793.3190465, "eval_loss": 3.402008295059204, "eval_runtime": 20.7141, "eval_samples_per_second": 459.929, "eval_steps_per_second": 0.483, "train/total_time_seconds": 170.76810985431075, "train/time_per_step_avg": 0.33903457280248406, "train/epoch_time_elapsed": 393.29645624384284, "train/estimated_remaining_minutes": 0.0}
+{"step": 500, "epoch": 0.016852039096730706, "timestamp": 1786569793.739768, "train_runtime": 394.5028, "train_samples_per_second": 40.557, "train_steps_per_second": 1.267, "total_flos": 1209853280256000.0, "train_loss": 4.914910415649414, "train/total_time_seconds": 170.76810985431075, "train/time_per_step_avg": 0.33903457280248406, "train/epoch_time_elapsed": 393.71717697381973, "train/estimated_remaining_minutes": 0.0}
+{"step": 500, "epoch": 0.016852039096730706, "timestamp": 1786569814.3469703, "eval_loss": 3.402008295059204, "eval_runtime": 20.6042, "eval_samples_per_second": 462.382, "eval_steps_per_second": 0.485, "train/total_time_seconds": 170.76810985431075, "train/time_per_step_avg": 0.33903457280248406, "train/epoch_time_elapsed": 414.324380222708, "train/estimated_remaining_minutes": 0.0}