Auto upload zain 2026-08-12T21:59:28.510912

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f131279a10fb420071daa72304fa36d4da15c3e9aea9606714109131ea9620af
+size 33967272

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e45f149026aa649e936f013c93edc058afc179a3569175da2844dc858c579d2
+size 68504996

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a281f149d9f0a359eec50d273883a2c794aa28cb3b5f6c98a39ab117debef17
+size 14244

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa8588bc4a247e7962c4619ae7e7ba14fb542cc23b88b7c743a799982e027413
+size 1064

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/trainer_state.json

@@ -0,0 +1,1003 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.12807549713515334,
+  "eval_steps": 50,
+  "global_step": 1900,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29962387084961,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047650909423828,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 7.835535049438477,
+      "eval_runtime": 15.2165,
+      "eval_samples_per_second": 626.095,
+      "eval_steps_per_second": 0.657,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.8378746032714846,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5728271484375,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.2332916259765625,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 7.039053440093994,
+      "eval_runtime": 15.2074,
+      "eval_samples_per_second": 626.471,
+      "eval_steps_per_second": 0.658,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.8608039855957035,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.517842864990234,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 6.229617595672607,
+      "eval_runtime": 15.1978,
+      "eval_samples_per_second": 626.867,
+      "eval_steps_per_second": 0.658,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.0002226,
+      "loss": 6.2399169921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 6.036548614501953,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0002786,
+      "loss": 5.857078552246094,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.765108585357666,
+      "eval_runtime": 15.5271,
+      "eval_samples_per_second": 613.574,
+      "eval_steps_per_second": 0.644,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.680564117431641,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.0003346,
+      "loss": 5.472259902954102,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.240335464477539,
+      "eval_runtime": 15.2596,
+      "eval_samples_per_second": 624.33,
+      "eval_steps_per_second": 0.655,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.7421875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 5.24955940246582,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0003906,
+      "loss": 4.995759963989258,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.0004186,
+      "loss": 4.769353866577148,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.669629096984863,
+      "eval_runtime": 15.2595,
+      "eval_samples_per_second": 624.332,
+      "eval_steps_per_second": 0.655,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.0004466,
+      "loss": 4.569867324829102,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.389446258544922,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.231207847595215,
+      "eval_runtime": 15.2143,
+      "eval_samples_per_second": 626.188,
+      "eval_steps_per_second": 0.657,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.0005026,
+      "loss": 4.243848419189453,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0005306,
+      "loss": 4.1090648651123045,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.484375,
+      "learning_rate": 0.0005586,
+      "loss": 3.966172790527344,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 3.923773765563965,
+      "eval_runtime": 15.5308,
+      "eval_samples_per_second": 613.426,
+      "eval_steps_per_second": 0.644,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.0005866,
+      "loss": 3.8559597015380858,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.0006146,
+      "loss": 3.7943866729736326,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.6977553367614746,
+      "eval_runtime": 15.2001,
+      "eval_samples_per_second": 626.77,
+      "eval_steps_per_second": 0.658,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0006426,
+      "loss": 3.686443328857422,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006705999999999999,
+      "loss": 3.6123157501220704,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.0006986,
+      "loss": 3.542654800415039,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.513820171356201,
+      "eval_runtime": 15.2193,
+      "eval_samples_per_second": 625.983,
+      "eval_steps_per_second": 0.657,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.472199249267578,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.396291732788086,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.3581483364105225,
+      "eval_runtime": 15.3044,
+      "eval_samples_per_second": 622.502,
+      "eval_steps_per_second": 0.653,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.0007,
+      "loss": 3.347300720214844,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.2885902404785154,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.25723876953125,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.2229573726654053,
+      "eval_runtime": 15.2967,
+      "eval_samples_per_second": 622.812,
+      "eval_steps_per_second": 0.654,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.1902334213256838,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 3.167861557006836,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.120636463165283,
+      "eval_runtime": 15.3751,
+      "eval_samples_per_second": 619.639,
+      "eval_steps_per_second": 0.65,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.0007,
+      "loss": 3.1089324951171875,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 3.0828268051147463,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 3.0396488189697264,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.0276408195495605,
+      "eval_runtime": 15.2735,
+      "eval_samples_per_second": 623.761,
+      "eval_steps_per_second": 0.655,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.0034805297851563,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.0007,
+      "loss": 2.96556396484375,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 2.950009346008301,
+      "eval_runtime": 15.243,
+      "eval_samples_per_second": 625.009,
+      "eval_steps_per_second": 0.656,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.0007,
+      "loss": 2.950779151916504,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.956203079223633,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.9017578125,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 2.8844006061553955,
+      "eval_runtime": 15.2903,
+      "eval_samples_per_second": 623.073,
+      "eval_steps_per_second": 0.654,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0007,
+      "loss": 2.887515640258789,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0007,
+      "loss": 2.8556928634643555,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 2.821610450744629,
+      "eval_runtime": 15.2253,
+      "eval_samples_per_second": 625.733,
+      "eval_steps_per_second": 0.657,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.8130855560302734,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.790329170227051,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 2.7760162353515625,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 2.7661373615264893,
+      "eval_runtime": 15.2736,
+      "eval_samples_per_second": 623.755,
+      "eval_steps_per_second": 0.655,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 2.7462047576904296,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.4140625,
+      "learning_rate": 0.0007,
+      "loss": 2.732274627685547,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 2.713942289352417,
+      "eval_runtime": 15.6062,
+      "eval_samples_per_second": 610.463,
+      "eval_steps_per_second": 0.641,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.0007,
+      "loss": 2.705318069458008,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0007,
+      "loss": 2.6892818450927733,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0007,
+      "loss": 2.687115287780762,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.668647289276123,
+      "eval_runtime": 15.2597,
+      "eval_samples_per_second": 624.326,
+      "eval_steps_per_second": 0.655,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.6533071517944338,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.6390922546386717,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07077856420626896,
+      "eval_loss": 2.6291816234588623,
+      "eval_runtime": 15.2457,
+      "eval_samples_per_second": 624.896,
+      "eval_steps_per_second": 0.656,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 2.624461555480957,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.6044427871704103,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.597646713256836,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "eval_loss": 2.589696168899536,
+      "eval_runtime": 15.3967,
+      "eval_samples_per_second": 618.768,
+      "eval_steps_per_second": 0.649,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.561947250366211,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.390625,
+      "learning_rate": 0.0007,
+      "loss": 2.54862117767334,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "eval_loss": 2.559185743331909,
+      "eval_runtime": 15.2877,
+      "eval_samples_per_second": 623.181,
+      "eval_steps_per_second": 0.654,
+      "step": 1150
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.0007,
+      "loss": 2.5469717025756835,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.540451240539551,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.53134708404541,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 2.521665096282959,
+      "eval_runtime": 15.2822,
+      "eval_samples_per_second": 623.407,
+      "eval_steps_per_second": 0.654,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0007,
+      "loss": 2.513539123535156,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.5009738922119142,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08426019548365352,
+      "eval_loss": 2.489459753036499,
+      "eval_runtime": 15.2493,
+      "eval_samples_per_second": 624.749,
+      "eval_steps_per_second": 0.656,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.4894077301025392,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.4697439193725588,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.461294174194336,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "eval_loss": 2.4627816677093506,
+      "eval_runtime": 15.421,
+      "eval_samples_per_second": 617.795,
+      "eval_steps_per_second": 0.648,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.452680206298828,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0007,
+      "loss": 2.4396684646606444,
+      "step": 1340
+    },
+    {
+      "epoch": 0.0910010111223458,
+      "eval_loss": 2.437220335006714,
+      "eval_runtime": 15.3026,
+      "eval_samples_per_second": 622.576,
+      "eval_steps_per_second": 0.653,
+      "step": 1350
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.0007,
+      "loss": 2.4397760391235352,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.4158536911010744,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.406942367553711,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 2.4096601009368896,
+      "eval_runtime": 15.357,
+      "eval_samples_per_second": 620.368,
+      "eval_steps_per_second": 0.651,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.4161144256591798,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.392984390258789,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09774182676103808,
+      "eval_loss": 2.3937113285064697,
+      "eval_runtime": 15.3555,
+      "eval_samples_per_second": 620.43,
+      "eval_steps_per_second": 0.651,
+      "step": 1450
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0007,
+      "loss": 2.3820331573486326,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.3687986373901366,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.355032730102539,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "eval_loss": 2.3681657314300537,
+      "eval_runtime": 15.3336,
+      "eval_samples_per_second": 621.317,
+      "eval_steps_per_second": 0.652,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10246039770812268,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.3572179794311525,
+      "step": 1520
+    },
+    {
+      "epoch": 0.10380856083586114,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.345840835571289,
+      "step": 1540
+    },
+    {
+      "epoch": 0.10448264239973036,
+      "eval_loss": 2.3475193977355957,
+      "eval_runtime": 15.3453,
+      "eval_samples_per_second": 620.842,
+      "eval_steps_per_second": 0.652,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1051567239635996,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.335231971740723,
+      "step": 1560
+    },
+    {
+      "epoch": 0.10650488709133805,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.3543283462524416,
+      "step": 1580
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.0007,
+      "loss": 2.3237049102783205,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "eval_loss": 2.3377676010131836,
+      "eval_runtime": 15.2858,
+      "eval_samples_per_second": 623.259,
+      "eval_steps_per_second": 0.654,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10920121334681497,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.3196277618408203,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11054937647455342,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.0007,
+      "loss": 2.3308162689208984,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11122345803842265,
+      "eval_loss": 2.313627004623413,
+      "eval_runtime": 15.276,
+      "eval_samples_per_second": 623.659,
+      "eval_steps_per_second": 0.655,
+      "step": 1650
+    },
+    {
+      "epoch": 0.11189753960229187,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.3104904174804686,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11324570273003033,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2955816268920897,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.3044401168823243,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "eval_loss": 2.2976956367492676,
+      "eval_runtime": 15.3404,
+      "eval_samples_per_second": 621.039,
+      "eval_steps_per_second": 0.652,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11594202898550725,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.280081939697266,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1172901921132457,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.269576644897461,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11796427367711493,
+      "eval_loss": 2.28181791305542,
+      "eval_runtime": 15.2137,
+      "eval_samples_per_second": 626.211,
+      "eval_steps_per_second": 0.657,
+      "step": 1750
+    },
+    {
+      "epoch": 0.11863835524098416,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.273081398010254,
+      "step": 1760
+    },
+    {
+      "epoch": 0.11998651836872262,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.2877025604248047,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2828989028930664,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "eval_loss": 2.2671334743499756,
+      "eval_runtime": 15.1615,
+      "eval_samples_per_second": 628.369,
+      "eval_steps_per_second": 0.66,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12268284462419952,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2743486404418944,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2429489135742187,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12470508931580722,
+      "eval_loss": 2.252441644668579,
+      "eval_runtime": 15.3255,
+      "eval_samples_per_second": 621.643,
+      "eval_steps_per_second": 0.653,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12537917087967643,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.242429733276367,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1267273340074149,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2348007202148437,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0007,
+      "loss": 2.241002655029297,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "eval_loss": 2.2387285232543945,
+      "eval_runtime": 15.2942,
+      "eval_samples_per_second": 622.915,
+      "eval_steps_per_second": 0.654,
+      "step": 1900
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6129939224985600.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-1900/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd90512e56f813612dc668676d6f8cab72da110a42f23c61cafebf91064625a1
+size 4920

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:438c7986cda0b16ba222df99a1b9a71e1e61a59e31c33be8b9bbab7476b80b38
+size 33967272

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eef39ef71ae89abf3a67ffd6f0086cfe43740f0e1cc8bce575ac13fc5d7c29e
+size 68504996

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eaeafa6ce3d508c1f46be3194994b96a35c312b7108bbff5a77daad15dca3f7
+size 14244

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb897171d6314e0d5ff47840624a03ce1078f22b8f8c3062289ef88ef22a406e
+size 1064

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/trainer_state.json

@@ -0,0 +1,1054 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.13481631277384565,
+  "eval_steps": 50,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29962387084961,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047650909423828,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 7.835535049438477,
+      "eval_runtime": 15.2165,
+      "eval_samples_per_second": 626.095,
+      "eval_steps_per_second": 0.657,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.8378746032714846,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5728271484375,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.2332916259765625,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 7.039053440093994,
+      "eval_runtime": 15.2074,
+      "eval_samples_per_second": 626.471,
+      "eval_steps_per_second": 0.658,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.8608039855957035,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.517842864990234,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 6.229617595672607,
+      "eval_runtime": 15.1978,
+      "eval_samples_per_second": 626.867,
+      "eval_steps_per_second": 0.658,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.0002226,
+      "loss": 6.2399169921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 6.036548614501953,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0002786,
+      "loss": 5.857078552246094,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.765108585357666,
+      "eval_runtime": 15.5271,
+      "eval_samples_per_second": 613.574,
+      "eval_steps_per_second": 0.644,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.680564117431641,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.0003346,
+      "loss": 5.472259902954102,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.240335464477539,
+      "eval_runtime": 15.2596,
+      "eval_samples_per_second": 624.33,
+      "eval_steps_per_second": 0.655,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.7421875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 5.24955940246582,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0003906,
+      "loss": 4.995759963989258,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.0004186,
+      "loss": 4.769353866577148,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.669629096984863,
+      "eval_runtime": 15.2595,
+      "eval_samples_per_second": 624.332,
+      "eval_steps_per_second": 0.655,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.0004466,
+      "loss": 4.569867324829102,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.389446258544922,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.231207847595215,
+      "eval_runtime": 15.2143,
+      "eval_samples_per_second": 626.188,
+      "eval_steps_per_second": 0.657,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.0005026,
+      "loss": 4.243848419189453,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0005306,
+      "loss": 4.1090648651123045,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.484375,
+      "learning_rate": 0.0005586,
+      "loss": 3.966172790527344,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 3.923773765563965,
+      "eval_runtime": 15.5308,
+      "eval_samples_per_second": 613.426,
+      "eval_steps_per_second": 0.644,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.0005866,
+      "loss": 3.8559597015380858,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.0006146,
+      "loss": 3.7943866729736326,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.6977553367614746,
+      "eval_runtime": 15.2001,
+      "eval_samples_per_second": 626.77,
+      "eval_steps_per_second": 0.658,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0006426,
+      "loss": 3.686443328857422,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006705999999999999,
+      "loss": 3.6123157501220704,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.0006986,
+      "loss": 3.542654800415039,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.513820171356201,
+      "eval_runtime": 15.2193,
+      "eval_samples_per_second": 625.983,
+      "eval_steps_per_second": 0.657,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.472199249267578,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.396291732788086,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.3581483364105225,
+      "eval_runtime": 15.3044,
+      "eval_samples_per_second": 622.502,
+      "eval_steps_per_second": 0.653,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.0007,
+      "loss": 3.347300720214844,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.2885902404785154,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.25723876953125,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.2229573726654053,
+      "eval_runtime": 15.2967,
+      "eval_samples_per_second": 622.812,
+      "eval_steps_per_second": 0.654,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.1902334213256838,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 3.167861557006836,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.120636463165283,
+      "eval_runtime": 15.3751,
+      "eval_samples_per_second": 619.639,
+      "eval_steps_per_second": 0.65,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.0007,
+      "loss": 3.1089324951171875,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 3.0828268051147463,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 3.0396488189697264,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.0276408195495605,
+      "eval_runtime": 15.2735,
+      "eval_samples_per_second": 623.761,
+      "eval_steps_per_second": 0.655,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.0034805297851563,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.0007,
+      "loss": 2.96556396484375,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 2.950009346008301,
+      "eval_runtime": 15.243,
+      "eval_samples_per_second": 625.009,
+      "eval_steps_per_second": 0.656,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.0007,
+      "loss": 2.950779151916504,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.956203079223633,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.9017578125,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 2.8844006061553955,
+      "eval_runtime": 15.2903,
+      "eval_samples_per_second": 623.073,
+      "eval_steps_per_second": 0.654,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0007,
+      "loss": 2.887515640258789,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0007,
+      "loss": 2.8556928634643555,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 2.821610450744629,
+      "eval_runtime": 15.2253,
+      "eval_samples_per_second": 625.733,
+      "eval_steps_per_second": 0.657,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.8130855560302734,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.790329170227051,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 2.7760162353515625,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 2.7661373615264893,
+      "eval_runtime": 15.2736,
+      "eval_samples_per_second": 623.755,
+      "eval_steps_per_second": 0.655,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 2.7462047576904296,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.4140625,
+      "learning_rate": 0.0007,
+      "loss": 2.732274627685547,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 2.713942289352417,
+      "eval_runtime": 15.6062,
+      "eval_samples_per_second": 610.463,
+      "eval_steps_per_second": 0.641,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.0007,
+      "loss": 2.705318069458008,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0007,
+      "loss": 2.6892818450927733,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0007,
+      "loss": 2.687115287780762,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.668647289276123,
+      "eval_runtime": 15.2597,
+      "eval_samples_per_second": 624.326,
+      "eval_steps_per_second": 0.655,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.6533071517944338,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.6390922546386717,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07077856420626896,
+      "eval_loss": 2.6291816234588623,
+      "eval_runtime": 15.2457,
+      "eval_samples_per_second": 624.896,
+      "eval_steps_per_second": 0.656,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 2.624461555480957,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.6044427871704103,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.597646713256836,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "eval_loss": 2.589696168899536,
+      "eval_runtime": 15.3967,
+      "eval_samples_per_second": 618.768,
+      "eval_steps_per_second": 0.649,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.561947250366211,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.390625,
+      "learning_rate": 0.0007,
+      "loss": 2.54862117767334,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "eval_loss": 2.559185743331909,
+      "eval_runtime": 15.2877,
+      "eval_samples_per_second": 623.181,
+      "eval_steps_per_second": 0.654,
+      "step": 1150
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.0007,
+      "loss": 2.5469717025756835,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.540451240539551,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.53134708404541,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 2.521665096282959,
+      "eval_runtime": 15.2822,
+      "eval_samples_per_second": 623.407,
+      "eval_steps_per_second": 0.654,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0007,
+      "loss": 2.513539123535156,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.5009738922119142,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08426019548365352,
+      "eval_loss": 2.489459753036499,
+      "eval_runtime": 15.2493,
+      "eval_samples_per_second": 624.749,
+      "eval_steps_per_second": 0.656,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.4894077301025392,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.4697439193725588,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.461294174194336,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "eval_loss": 2.4627816677093506,
+      "eval_runtime": 15.421,
+      "eval_samples_per_second": 617.795,
+      "eval_steps_per_second": 0.648,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.452680206298828,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0007,
+      "loss": 2.4396684646606444,
+      "step": 1340
+    },
+    {
+      "epoch": 0.0910010111223458,
+      "eval_loss": 2.437220335006714,
+      "eval_runtime": 15.3026,
+      "eval_samples_per_second": 622.576,
+      "eval_steps_per_second": 0.653,
+      "step": 1350
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.0007,
+      "loss": 2.4397760391235352,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.4158536911010744,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.406942367553711,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 2.4096601009368896,
+      "eval_runtime": 15.357,
+      "eval_samples_per_second": 620.368,
+      "eval_steps_per_second": 0.651,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.4161144256591798,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.392984390258789,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09774182676103808,
+      "eval_loss": 2.3937113285064697,
+      "eval_runtime": 15.3555,
+      "eval_samples_per_second": 620.43,
+      "eval_steps_per_second": 0.651,
+      "step": 1450
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0007,
+      "loss": 2.3820331573486326,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.3687986373901366,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.355032730102539,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "eval_loss": 2.3681657314300537,
+      "eval_runtime": 15.3336,
+      "eval_samples_per_second": 621.317,
+      "eval_steps_per_second": 0.652,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10246039770812268,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.3572179794311525,
+      "step": 1520
+    },
+    {
+      "epoch": 0.10380856083586114,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.345840835571289,
+      "step": 1540
+    },
+    {
+      "epoch": 0.10448264239973036,
+      "eval_loss": 2.3475193977355957,
+      "eval_runtime": 15.3453,
+      "eval_samples_per_second": 620.842,
+      "eval_steps_per_second": 0.652,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1051567239635996,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.335231971740723,
+      "step": 1560
+    },
+    {
+      "epoch": 0.10650488709133805,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.3543283462524416,
+      "step": 1580
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.0007,
+      "loss": 2.3237049102783205,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "eval_loss": 2.3377676010131836,
+      "eval_runtime": 15.2858,
+      "eval_samples_per_second": 623.259,
+      "eval_steps_per_second": 0.654,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10920121334681497,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.3196277618408203,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11054937647455342,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.0007,
+      "loss": 2.3308162689208984,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11122345803842265,
+      "eval_loss": 2.313627004623413,
+      "eval_runtime": 15.276,
+      "eval_samples_per_second": 623.659,
+      "eval_steps_per_second": 0.655,
+      "step": 1650
+    },
+    {
+      "epoch": 0.11189753960229187,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.3104904174804686,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11324570273003033,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2955816268920897,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.3044401168823243,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "eval_loss": 2.2976956367492676,
+      "eval_runtime": 15.3404,
+      "eval_samples_per_second": 621.039,
+      "eval_steps_per_second": 0.652,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11594202898550725,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.280081939697266,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1172901921132457,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.269576644897461,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11796427367711493,
+      "eval_loss": 2.28181791305542,
+      "eval_runtime": 15.2137,
+      "eval_samples_per_second": 626.211,
+      "eval_steps_per_second": 0.657,
+      "step": 1750
+    },
+    {
+      "epoch": 0.11863835524098416,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.273081398010254,
+      "step": 1760
+    },
+    {
+      "epoch": 0.11998651836872262,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.2877025604248047,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2828989028930664,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "eval_loss": 2.2671334743499756,
+      "eval_runtime": 15.1615,
+      "eval_samples_per_second": 628.369,
+      "eval_steps_per_second": 0.66,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12268284462419952,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2743486404418944,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2429489135742187,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12470508931580722,
+      "eval_loss": 2.252441644668579,
+      "eval_runtime": 15.3255,
+      "eval_samples_per_second": 621.643,
+      "eval_steps_per_second": 0.653,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12537917087967643,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.242429733276367,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1267273340074149,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2348007202148437,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0007,
+      "loss": 2.241002655029297,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "eval_loss": 2.2387285232543945,
+      "eval_runtime": 15.2942,
+      "eval_samples_per_second": 622.915,
+      "eval_steps_per_second": 0.654,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12942366026289182,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.0007,
+      "loss": 2.2172693252563476,
+      "step": 1920
+    },
+    {
+      "epoch": 0.13077182339063026,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.2283090591430663,
+      "step": 1940
+    },
+    {
+      "epoch": 0.13144590495449948,
+      "eval_loss": 2.225919246673584,
+      "eval_runtime": 15.3259,
+      "eval_samples_per_second": 621.627,
+      "eval_steps_per_second": 0.652,
+      "step": 1950
+    },
+    {
+      "epoch": 0.13211998651836873,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0007,
+      "loss": 2.2027830123901366,
+      "step": 1960
+    },
+    {
+      "epoch": 0.13346814964610718,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.224252128601074,
+      "step": 1980
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.1990388870239257,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "eval_loss": 2.215721368789673,
+      "eval_runtime": 15.3022,
+      "eval_samples_per_second": 622.59,
+      "eval_steps_per_second": 0.654,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6452567605248000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd90512e56f813612dc668676d6f8cab72da110a42f23c61cafebf91064625a1
+size 4920

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f80665fb3305e295dac85dc188657b5af87deae6e14c49d7bc8660847ff8b1b3
+size 33967272

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68718ced8b2991f4dc287310d13f9890123b65355b9749771b4264d987395f11
+size 68504996

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181ce3e2aecd11b7beeabbdbd7d59126216a3702cd0496a6ee8850d8e39aa35b
+size 14244

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:985eadbe03d32df01f608a67ee99d09997f8b557ed94b6768e8445a622bb3975
+size 1064

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/trainer_state.json

@@ -0,0 +1,1105 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.14155712841253792,
+  "eval_steps": 50,
+  "global_step": 2100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29962387084961,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047650909423828,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 7.835535049438477,
+      "eval_runtime": 15.2165,
+      "eval_samples_per_second": 626.095,
+      "eval_steps_per_second": 0.657,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.8378746032714846,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5728271484375,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.2332916259765625,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 7.039053440093994,
+      "eval_runtime": 15.2074,
+      "eval_samples_per_second": 626.471,
+      "eval_steps_per_second": 0.658,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.8608039855957035,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.517842864990234,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 6.229617595672607,
+      "eval_runtime": 15.1978,
+      "eval_samples_per_second": 626.867,
+      "eval_steps_per_second": 0.658,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.0002226,
+      "loss": 6.2399169921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 6.036548614501953,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0002786,
+      "loss": 5.857078552246094,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.765108585357666,
+      "eval_runtime": 15.5271,
+      "eval_samples_per_second": 613.574,
+      "eval_steps_per_second": 0.644,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.680564117431641,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.0003346,
+      "loss": 5.472259902954102,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.240335464477539,
+      "eval_runtime": 15.2596,
+      "eval_samples_per_second": 624.33,
+      "eval_steps_per_second": 0.655,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.7421875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 5.24955940246582,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0003906,
+      "loss": 4.995759963989258,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.0004186,
+      "loss": 4.769353866577148,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.669629096984863,
+      "eval_runtime": 15.2595,
+      "eval_samples_per_second": 624.332,
+      "eval_steps_per_second": 0.655,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.0004466,
+      "loss": 4.569867324829102,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.389446258544922,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.231207847595215,
+      "eval_runtime": 15.2143,
+      "eval_samples_per_second": 626.188,
+      "eval_steps_per_second": 0.657,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.0005026,
+      "loss": 4.243848419189453,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0005306,
+      "loss": 4.1090648651123045,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.484375,
+      "learning_rate": 0.0005586,
+      "loss": 3.966172790527344,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 3.923773765563965,
+      "eval_runtime": 15.5308,
+      "eval_samples_per_second": 613.426,
+      "eval_steps_per_second": 0.644,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.0005866,
+      "loss": 3.8559597015380858,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.0006146,
+      "loss": 3.7943866729736326,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.6977553367614746,
+      "eval_runtime": 15.2001,
+      "eval_samples_per_second": 626.77,
+      "eval_steps_per_second": 0.658,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0006426,
+      "loss": 3.686443328857422,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006705999999999999,
+      "loss": 3.6123157501220704,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.0006986,
+      "loss": 3.542654800415039,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.513820171356201,
+      "eval_runtime": 15.2193,
+      "eval_samples_per_second": 625.983,
+      "eval_steps_per_second": 0.657,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.472199249267578,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.396291732788086,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.3581483364105225,
+      "eval_runtime": 15.3044,
+      "eval_samples_per_second": 622.502,
+      "eval_steps_per_second": 0.653,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.0007,
+      "loss": 3.347300720214844,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.2885902404785154,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.25723876953125,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.2229573726654053,
+      "eval_runtime": 15.2967,
+      "eval_samples_per_second": 622.812,
+      "eval_steps_per_second": 0.654,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.1902334213256838,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 3.167861557006836,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.120636463165283,
+      "eval_runtime": 15.3751,
+      "eval_samples_per_second": 619.639,
+      "eval_steps_per_second": 0.65,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.0007,
+      "loss": 3.1089324951171875,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 3.0828268051147463,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 3.0396488189697264,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.0276408195495605,
+      "eval_runtime": 15.2735,
+      "eval_samples_per_second": 623.761,
+      "eval_steps_per_second": 0.655,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.0034805297851563,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.0007,
+      "loss": 2.96556396484375,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 2.950009346008301,
+      "eval_runtime": 15.243,
+      "eval_samples_per_second": 625.009,
+      "eval_steps_per_second": 0.656,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.0007,
+      "loss": 2.950779151916504,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.956203079223633,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.9017578125,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 2.8844006061553955,
+      "eval_runtime": 15.2903,
+      "eval_samples_per_second": 623.073,
+      "eval_steps_per_second": 0.654,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0007,
+      "loss": 2.887515640258789,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0007,
+      "loss": 2.8556928634643555,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 2.821610450744629,
+      "eval_runtime": 15.2253,
+      "eval_samples_per_second": 625.733,
+      "eval_steps_per_second": 0.657,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.8130855560302734,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.790329170227051,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 2.7760162353515625,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 2.7661373615264893,
+      "eval_runtime": 15.2736,
+      "eval_samples_per_second": 623.755,
+      "eval_steps_per_second": 0.655,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 2.7462047576904296,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.4140625,
+      "learning_rate": 0.0007,
+      "loss": 2.732274627685547,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 2.713942289352417,
+      "eval_runtime": 15.6062,
+      "eval_samples_per_second": 610.463,
+      "eval_steps_per_second": 0.641,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.0007,
+      "loss": 2.705318069458008,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0007,
+      "loss": 2.6892818450927733,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0007,
+      "loss": 2.687115287780762,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.668647289276123,
+      "eval_runtime": 15.2597,
+      "eval_samples_per_second": 624.326,
+      "eval_steps_per_second": 0.655,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.6533071517944338,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.6390922546386717,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07077856420626896,
+      "eval_loss": 2.6291816234588623,
+      "eval_runtime": 15.2457,
+      "eval_samples_per_second": 624.896,
+      "eval_steps_per_second": 0.656,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 2.624461555480957,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.6044427871704103,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.597646713256836,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "eval_loss": 2.589696168899536,
+      "eval_runtime": 15.3967,
+      "eval_samples_per_second": 618.768,
+      "eval_steps_per_second": 0.649,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.561947250366211,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.390625,
+      "learning_rate": 0.0007,
+      "loss": 2.54862117767334,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "eval_loss": 2.559185743331909,
+      "eval_runtime": 15.2877,
+      "eval_samples_per_second": 623.181,
+      "eval_steps_per_second": 0.654,
+      "step": 1150
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.0007,
+      "loss": 2.5469717025756835,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.540451240539551,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.53134708404541,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 2.521665096282959,
+      "eval_runtime": 15.2822,
+      "eval_samples_per_second": 623.407,
+      "eval_steps_per_second": 0.654,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0007,
+      "loss": 2.513539123535156,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.5009738922119142,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08426019548365352,
+      "eval_loss": 2.489459753036499,
+      "eval_runtime": 15.2493,
+      "eval_samples_per_second": 624.749,
+      "eval_steps_per_second": 0.656,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.4894077301025392,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.4697439193725588,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.461294174194336,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "eval_loss": 2.4627816677093506,
+      "eval_runtime": 15.421,
+      "eval_samples_per_second": 617.795,
+      "eval_steps_per_second": 0.648,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.452680206298828,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0007,
+      "loss": 2.4396684646606444,
+      "step": 1340
+    },
+    {
+      "epoch": 0.0910010111223458,
+      "eval_loss": 2.437220335006714,
+      "eval_runtime": 15.3026,
+      "eval_samples_per_second": 622.576,
+      "eval_steps_per_second": 0.653,
+      "step": 1350
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.0007,
+      "loss": 2.4397760391235352,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.4158536911010744,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.406942367553711,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 2.4096601009368896,
+      "eval_runtime": 15.357,
+      "eval_samples_per_second": 620.368,
+      "eval_steps_per_second": 0.651,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.4161144256591798,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.392984390258789,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09774182676103808,
+      "eval_loss": 2.3937113285064697,
+      "eval_runtime": 15.3555,
+      "eval_samples_per_second": 620.43,
+      "eval_steps_per_second": 0.651,
+      "step": 1450
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0007,
+      "loss": 2.3820331573486326,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.3687986373901366,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.355032730102539,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "eval_loss": 2.3681657314300537,
+      "eval_runtime": 15.3336,
+      "eval_samples_per_second": 621.317,
+      "eval_steps_per_second": 0.652,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10246039770812268,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.3572179794311525,
+      "step": 1520
+    },
+    {
+      "epoch": 0.10380856083586114,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.345840835571289,
+      "step": 1540
+    },
+    {
+      "epoch": 0.10448264239973036,
+      "eval_loss": 2.3475193977355957,
+      "eval_runtime": 15.3453,
+      "eval_samples_per_second": 620.842,
+      "eval_steps_per_second": 0.652,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1051567239635996,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.335231971740723,
+      "step": 1560
+    },
+    {
+      "epoch": 0.10650488709133805,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.3543283462524416,
+      "step": 1580
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.0007,
+      "loss": 2.3237049102783205,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "eval_loss": 2.3377676010131836,
+      "eval_runtime": 15.2858,
+      "eval_samples_per_second": 623.259,
+      "eval_steps_per_second": 0.654,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10920121334681497,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.3196277618408203,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11054937647455342,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.0007,
+      "loss": 2.3308162689208984,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11122345803842265,
+      "eval_loss": 2.313627004623413,
+      "eval_runtime": 15.276,
+      "eval_samples_per_second": 623.659,
+      "eval_steps_per_second": 0.655,
+      "step": 1650
+    },
+    {
+      "epoch": 0.11189753960229187,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.3104904174804686,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11324570273003033,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2955816268920897,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.3044401168823243,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "eval_loss": 2.2976956367492676,
+      "eval_runtime": 15.3404,
+      "eval_samples_per_second": 621.039,
+      "eval_steps_per_second": 0.652,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11594202898550725,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.280081939697266,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1172901921132457,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.269576644897461,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11796427367711493,
+      "eval_loss": 2.28181791305542,
+      "eval_runtime": 15.2137,
+      "eval_samples_per_second": 626.211,
+      "eval_steps_per_second": 0.657,
+      "step": 1750
+    },
+    {
+      "epoch": 0.11863835524098416,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.273081398010254,
+      "step": 1760
+    },
+    {
+      "epoch": 0.11998651836872262,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.2877025604248047,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2828989028930664,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "eval_loss": 2.2671334743499756,
+      "eval_runtime": 15.1615,
+      "eval_samples_per_second": 628.369,
+      "eval_steps_per_second": 0.66,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12268284462419952,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2743486404418944,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2429489135742187,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12470508931580722,
+      "eval_loss": 2.252441644668579,
+      "eval_runtime": 15.3255,
+      "eval_samples_per_second": 621.643,
+      "eval_steps_per_second": 0.653,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12537917087967643,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.242429733276367,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1267273340074149,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2348007202148437,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0007,
+      "loss": 2.241002655029297,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "eval_loss": 2.2387285232543945,
+      "eval_runtime": 15.2942,
+      "eval_samples_per_second": 622.915,
+      "eval_steps_per_second": 0.654,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12942366026289182,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.0007,
+      "loss": 2.2172693252563476,
+      "step": 1920
+    },
+    {
+      "epoch": 0.13077182339063026,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.2283090591430663,
+      "step": 1940
+    },
+    {
+      "epoch": 0.13144590495449948,
+      "eval_loss": 2.225919246673584,
+      "eval_runtime": 15.3259,
+      "eval_samples_per_second": 621.627,
+      "eval_steps_per_second": 0.652,
+      "step": 1950
+    },
+    {
+      "epoch": 0.13211998651836873,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0007,
+      "loss": 2.2027830123901366,
+      "step": 1960
+    },
+    {
+      "epoch": 0.13346814964610718,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.224252128601074,
+      "step": 1980
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.1990388870239257,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "eval_loss": 2.215721368789673,
+      "eval_runtime": 15.3022,
+      "eval_samples_per_second": 622.59,
+      "eval_steps_per_second": 0.654,
+      "step": 2000
+    },
+    {
+      "epoch": 0.1361644759015841,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.207795524597168,
+      "step": 2020
+    },
+    {
+      "epoch": 0.13751263902932254,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.1882051467895507,
+      "step": 2040
+    },
+    {
+      "epoch": 0.1381867205931918,
+      "eval_loss": 2.2054405212402344,
+      "eval_runtime": 15.4183,
+      "eval_samples_per_second": 617.904,
+      "eval_steps_per_second": 0.649,
+      "step": 2050
+    },
+    {
+      "epoch": 0.138860802157061,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0007,
+      "loss": 2.206527900695801,
+      "step": 2060
+    },
+    {
+      "epoch": 0.14020896528479945,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0007,
+      "loss": 2.1837987899780273,
+      "step": 2080
+    },
+    {
+      "epoch": 0.14155712841253792,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0007,
+      "loss": 2.1744739532470705,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14155712841253792,
+      "eval_loss": 2.1948788166046143,
+      "eval_runtime": 15.1797,
+      "eval_samples_per_second": 627.615,
+      "eval_steps_per_second": 0.659,
+      "step": 2100
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6775195985510400.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd90512e56f813612dc668676d6f8cab72da110a42f23c61cafebf91064625a1
+size 4920

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f698c03047f51fe577b8fb3453aeb08f5d6de316653a9b8bd7e128bd28e2075c
+size 33967272

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7528b3ab660081ae753dfed81b3128259147180d964a7bf9fde194680b1aed99
+size 68504996

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61fafc0c97467ebf3ffe1a393e362f674f853cc5aa2ccec4c50a85214629a8ad
+size 14244

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb38d8fe479c6ce36db4ed357fb0375cafd5d59c32c20acad02585958763071c
+size 1064

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/trainer_state.json

@@ -0,0 +1,1156 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1482979440512302,
+  "eval_steps": 50,
+  "global_step": 2200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29962387084961,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047650909423828,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 7.835535049438477,
+      "eval_runtime": 15.2165,
+      "eval_samples_per_second": 626.095,
+      "eval_steps_per_second": 0.657,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.8378746032714846,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5728271484375,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.2332916259765625,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 7.039053440093994,
+      "eval_runtime": 15.2074,
+      "eval_samples_per_second": 626.471,
+      "eval_steps_per_second": 0.658,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.8608039855957035,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.517842864990234,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 6.229617595672607,
+      "eval_runtime": 15.1978,
+      "eval_samples_per_second": 626.867,
+      "eval_steps_per_second": 0.658,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.0002226,
+      "loss": 6.2399169921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 6.036548614501953,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0002786,
+      "loss": 5.857078552246094,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.765108585357666,
+      "eval_runtime": 15.5271,
+      "eval_samples_per_second": 613.574,
+      "eval_steps_per_second": 0.644,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.680564117431641,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.0003346,
+      "loss": 5.472259902954102,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.240335464477539,
+      "eval_runtime": 15.2596,
+      "eval_samples_per_second": 624.33,
+      "eval_steps_per_second": 0.655,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.7421875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 5.24955940246582,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0003906,
+      "loss": 4.995759963989258,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.0004186,
+      "loss": 4.769353866577148,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.669629096984863,
+      "eval_runtime": 15.2595,
+      "eval_samples_per_second": 624.332,
+      "eval_steps_per_second": 0.655,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.0004466,
+      "loss": 4.569867324829102,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.389446258544922,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.231207847595215,
+      "eval_runtime": 15.2143,
+      "eval_samples_per_second": 626.188,
+      "eval_steps_per_second": 0.657,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.0005026,
+      "loss": 4.243848419189453,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0005306,
+      "loss": 4.1090648651123045,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.484375,
+      "learning_rate": 0.0005586,
+      "loss": 3.966172790527344,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 3.923773765563965,
+      "eval_runtime": 15.5308,
+      "eval_samples_per_second": 613.426,
+      "eval_steps_per_second": 0.644,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.0005866,
+      "loss": 3.8559597015380858,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.0006146,
+      "loss": 3.7943866729736326,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.6977553367614746,
+      "eval_runtime": 15.2001,
+      "eval_samples_per_second": 626.77,
+      "eval_steps_per_second": 0.658,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0006426,
+      "loss": 3.686443328857422,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006705999999999999,
+      "loss": 3.6123157501220704,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.0006986,
+      "loss": 3.542654800415039,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.513820171356201,
+      "eval_runtime": 15.2193,
+      "eval_samples_per_second": 625.983,
+      "eval_steps_per_second": 0.657,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.472199249267578,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.396291732788086,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.3581483364105225,
+      "eval_runtime": 15.3044,
+      "eval_samples_per_second": 622.502,
+      "eval_steps_per_second": 0.653,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.0007,
+      "loss": 3.347300720214844,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.2885902404785154,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.25723876953125,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.2229573726654053,
+      "eval_runtime": 15.2967,
+      "eval_samples_per_second": 622.812,
+      "eval_steps_per_second": 0.654,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.1902334213256838,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 3.167861557006836,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.120636463165283,
+      "eval_runtime": 15.3751,
+      "eval_samples_per_second": 619.639,
+      "eval_steps_per_second": 0.65,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.0007,
+      "loss": 3.1089324951171875,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 3.0828268051147463,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 3.0396488189697264,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.0276408195495605,
+      "eval_runtime": 15.2735,
+      "eval_samples_per_second": 623.761,
+      "eval_steps_per_second": 0.655,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.0034805297851563,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.0007,
+      "loss": 2.96556396484375,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 2.950009346008301,
+      "eval_runtime": 15.243,
+      "eval_samples_per_second": 625.009,
+      "eval_steps_per_second": 0.656,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.0007,
+      "loss": 2.950779151916504,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.956203079223633,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.9017578125,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 2.8844006061553955,
+      "eval_runtime": 15.2903,
+      "eval_samples_per_second": 623.073,
+      "eval_steps_per_second": 0.654,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0007,
+      "loss": 2.887515640258789,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0007,
+      "loss": 2.8556928634643555,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 2.821610450744629,
+      "eval_runtime": 15.2253,
+      "eval_samples_per_second": 625.733,
+      "eval_steps_per_second": 0.657,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.8130855560302734,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.790329170227051,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 2.7760162353515625,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 2.7661373615264893,
+      "eval_runtime": 15.2736,
+      "eval_samples_per_second": 623.755,
+      "eval_steps_per_second": 0.655,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 2.7462047576904296,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.4140625,
+      "learning_rate": 0.0007,
+      "loss": 2.732274627685547,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 2.713942289352417,
+      "eval_runtime": 15.6062,
+      "eval_samples_per_second": 610.463,
+      "eval_steps_per_second": 0.641,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.0007,
+      "loss": 2.705318069458008,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0007,
+      "loss": 2.6892818450927733,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0007,
+      "loss": 2.687115287780762,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.668647289276123,
+      "eval_runtime": 15.2597,
+      "eval_samples_per_second": 624.326,
+      "eval_steps_per_second": 0.655,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.6533071517944338,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.6390922546386717,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07077856420626896,
+      "eval_loss": 2.6291816234588623,
+      "eval_runtime": 15.2457,
+      "eval_samples_per_second": 624.896,
+      "eval_steps_per_second": 0.656,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 2.624461555480957,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.6044427871704103,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.597646713256836,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "eval_loss": 2.589696168899536,
+      "eval_runtime": 15.3967,
+      "eval_samples_per_second": 618.768,
+      "eval_steps_per_second": 0.649,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.561947250366211,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.390625,
+      "learning_rate": 0.0007,
+      "loss": 2.54862117767334,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "eval_loss": 2.559185743331909,
+      "eval_runtime": 15.2877,
+      "eval_samples_per_second": 623.181,
+      "eval_steps_per_second": 0.654,
+      "step": 1150
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.0007,
+      "loss": 2.5469717025756835,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.540451240539551,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.53134708404541,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 2.521665096282959,
+      "eval_runtime": 15.2822,
+      "eval_samples_per_second": 623.407,
+      "eval_steps_per_second": 0.654,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0007,
+      "loss": 2.513539123535156,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.5009738922119142,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08426019548365352,
+      "eval_loss": 2.489459753036499,
+      "eval_runtime": 15.2493,
+      "eval_samples_per_second": 624.749,
+      "eval_steps_per_second": 0.656,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.4894077301025392,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.4697439193725588,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.461294174194336,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "eval_loss": 2.4627816677093506,
+      "eval_runtime": 15.421,
+      "eval_samples_per_second": 617.795,
+      "eval_steps_per_second": 0.648,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.452680206298828,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0007,
+      "loss": 2.4396684646606444,
+      "step": 1340
+    },
+    {
+      "epoch": 0.0910010111223458,
+      "eval_loss": 2.437220335006714,
+      "eval_runtime": 15.3026,
+      "eval_samples_per_second": 622.576,
+      "eval_steps_per_second": 0.653,
+      "step": 1350
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.0007,
+      "loss": 2.4397760391235352,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.4158536911010744,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.406942367553711,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 2.4096601009368896,
+      "eval_runtime": 15.357,
+      "eval_samples_per_second": 620.368,
+      "eval_steps_per_second": 0.651,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.4161144256591798,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.392984390258789,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09774182676103808,
+      "eval_loss": 2.3937113285064697,
+      "eval_runtime": 15.3555,
+      "eval_samples_per_second": 620.43,
+      "eval_steps_per_second": 0.651,
+      "step": 1450
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0007,
+      "loss": 2.3820331573486326,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.3687986373901366,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.355032730102539,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "eval_loss": 2.3681657314300537,
+      "eval_runtime": 15.3336,
+      "eval_samples_per_second": 621.317,
+      "eval_steps_per_second": 0.652,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10246039770812268,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.3572179794311525,
+      "step": 1520
+    },
+    {
+      "epoch": 0.10380856083586114,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.345840835571289,
+      "step": 1540
+    },
+    {
+      "epoch": 0.10448264239973036,
+      "eval_loss": 2.3475193977355957,
+      "eval_runtime": 15.3453,
+      "eval_samples_per_second": 620.842,
+      "eval_steps_per_second": 0.652,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1051567239635996,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.335231971740723,
+      "step": 1560
+    },
+    {
+      "epoch": 0.10650488709133805,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.3543283462524416,
+      "step": 1580
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.0007,
+      "loss": 2.3237049102783205,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "eval_loss": 2.3377676010131836,
+      "eval_runtime": 15.2858,
+      "eval_samples_per_second": 623.259,
+      "eval_steps_per_second": 0.654,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10920121334681497,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.3196277618408203,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11054937647455342,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.0007,
+      "loss": 2.3308162689208984,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11122345803842265,
+      "eval_loss": 2.313627004623413,
+      "eval_runtime": 15.276,
+      "eval_samples_per_second": 623.659,
+      "eval_steps_per_second": 0.655,
+      "step": 1650
+    },
+    {
+      "epoch": 0.11189753960229187,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.3104904174804686,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11324570273003033,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2955816268920897,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.3044401168823243,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "eval_loss": 2.2976956367492676,
+      "eval_runtime": 15.3404,
+      "eval_samples_per_second": 621.039,
+      "eval_steps_per_second": 0.652,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11594202898550725,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.280081939697266,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1172901921132457,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.269576644897461,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11796427367711493,
+      "eval_loss": 2.28181791305542,
+      "eval_runtime": 15.2137,
+      "eval_samples_per_second": 626.211,
+      "eval_steps_per_second": 0.657,
+      "step": 1750
+    },
+    {
+      "epoch": 0.11863835524098416,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.273081398010254,
+      "step": 1760
+    },
+    {
+      "epoch": 0.11998651836872262,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.2877025604248047,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2828989028930664,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "eval_loss": 2.2671334743499756,
+      "eval_runtime": 15.1615,
+      "eval_samples_per_second": 628.369,
+      "eval_steps_per_second": 0.66,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12268284462419952,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2743486404418944,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2429489135742187,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12470508931580722,
+      "eval_loss": 2.252441644668579,
+      "eval_runtime": 15.3255,
+      "eval_samples_per_second": 621.643,
+      "eval_steps_per_second": 0.653,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12537917087967643,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.242429733276367,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1267273340074149,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2348007202148437,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0007,
+      "loss": 2.241002655029297,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "eval_loss": 2.2387285232543945,
+      "eval_runtime": 15.2942,
+      "eval_samples_per_second": 622.915,
+      "eval_steps_per_second": 0.654,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12942366026289182,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.0007,
+      "loss": 2.2172693252563476,
+      "step": 1920
+    },
+    {
+      "epoch": 0.13077182339063026,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.2283090591430663,
+      "step": 1940
+    },
+    {
+      "epoch": 0.13144590495449948,
+      "eval_loss": 2.225919246673584,
+      "eval_runtime": 15.3259,
+      "eval_samples_per_second": 621.627,
+      "eval_steps_per_second": 0.652,
+      "step": 1950
+    },
+    {
+      "epoch": 0.13211998651836873,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0007,
+      "loss": 2.2027830123901366,
+      "step": 1960
+    },
+    {
+      "epoch": 0.13346814964610718,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.224252128601074,
+      "step": 1980
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.1990388870239257,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "eval_loss": 2.215721368789673,
+      "eval_runtime": 15.3022,
+      "eval_samples_per_second": 622.59,
+      "eval_steps_per_second": 0.654,
+      "step": 2000
+    },
+    {
+      "epoch": 0.1361644759015841,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.207795524597168,
+      "step": 2020
+    },
+    {
+      "epoch": 0.13751263902932254,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.1882051467895507,
+      "step": 2040
+    },
+    {
+      "epoch": 0.1381867205931918,
+      "eval_loss": 2.2054405212402344,
+      "eval_runtime": 15.4183,
+      "eval_samples_per_second": 617.904,
+      "eval_steps_per_second": 0.649,
+      "step": 2050
+    },
+    {
+      "epoch": 0.138860802157061,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0007,
+      "loss": 2.206527900695801,
+      "step": 2060
+    },
+    {
+      "epoch": 0.14020896528479945,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0007,
+      "loss": 2.1837987899780273,
+      "step": 2080
+    },
+    {
+      "epoch": 0.14155712841253792,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0007,
+      "loss": 2.1744739532470705,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14155712841253792,
+      "eval_loss": 2.1948788166046143,
+      "eval_runtime": 15.1797,
+      "eval_samples_per_second": 627.615,
+      "eval_steps_per_second": 0.659,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14290529154027637,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0007,
+      "loss": 2.182137298583984,
+      "step": 2120
+    },
+    {
+      "epoch": 0.14425345466801484,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0007,
+      "loss": 2.1851844787597656,
+      "step": 2140
+    },
+    {
+      "epoch": 0.14492753623188406,
+      "eval_loss": 2.1849257946014404,
+      "eval_runtime": 15.2467,
+      "eval_samples_per_second": 624.858,
+      "eval_steps_per_second": 0.656,
+      "step": 2150
+    },
+    {
+      "epoch": 0.14560161779575329,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0007,
+      "loss": 2.1681228637695313,
+      "step": 2160
+    },
+    {
+      "epoch": 0.14694978092349173,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.0007,
+      "loss": 2.1777414321899413,
+      "step": 2180
+    },
+    {
+      "epoch": 0.1482979440512302,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0007,
+      "loss": 2.1520442962646484,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1482979440512302,
+      "eval_loss": 2.1730399131774902,
+      "eval_runtime": 15.3329,
+      "eval_samples_per_second": 621.344,
+      "eval_steps_per_second": 0.652,
+      "step": 2200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7097824365772800.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd90512e56f813612dc668676d6f8cab72da110a42f23c61cafebf91064625a1
+size 4920

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 100,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1246e2dbf801419985bcfa475e02980196f19e811b6e37653fab6efeb60d85cf
+size 33967272

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:103098e26979ff57050b00d826144440d4aa3d783a04968400b29a5dafc7e0c0
+size 68504996

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dec9f38518272c3edcef2f4b76e7cf3ba41857ee958849b9cec81d28afbeefdc
+size 14244

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b9bcfc4abc1cd7f19c4260bd52451981a53bf8db4a664a25c46ab7065952dd0
+size 1064

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/trainer_state.json

@@ -0,0 +1,1207 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.15503875968992248,
+  "eval_steps": 50,
+  "global_step": 2300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.66e-05,
+      "loss": 8.29962387084961,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.46e-05,
+      "loss": 8.047650909423828,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 7.835535049438477,
+      "eval_runtime": 15.2165,
+      "eval_samples_per_second": 626.095,
+      "eval_steps_per_second": 0.657,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 1.2734375,
+      "learning_rate": 8.259999999999999e-05,
+      "loss": 7.8378746032714846,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0001106,
+      "loss": 7.5728271484375,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.1953125,
+      "learning_rate": 0.0001386,
+      "loss": 7.2332916259765625,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 7.039053440093994,
+      "eval_runtime": 15.2074,
+      "eval_samples_per_second": 626.471,
+      "eval_steps_per_second": 0.658,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.00016659999999999998,
+      "loss": 6.8608039855957035,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00019460000000000001,
+      "loss": 6.517842864990234,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 6.229617595672607,
+      "eval_runtime": 15.1978,
+      "eval_samples_per_second": 626.867,
+      "eval_steps_per_second": 0.658,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.80078125,
+      "learning_rate": 0.0002226,
+      "loss": 6.2399169921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.00025059999999999997,
+      "loss": 6.036548614501953,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0002786,
+      "loss": 5.857078552246094,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.765108585357666,
+      "eval_runtime": 15.5271,
+      "eval_samples_per_second": 613.574,
+      "eval_steps_per_second": 0.644,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.00030659999999999997,
+      "loss": 5.680564117431641,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.0003346,
+      "loss": 5.472259902954102,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.240335464477539,
+      "eval_runtime": 15.2596,
+      "eval_samples_per_second": 624.33,
+      "eval_steps_per_second": 0.655,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.7421875,
+      "learning_rate": 0.00036260000000000003,
+      "loss": 5.24955940246582,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0003906,
+      "loss": 4.995759963989258,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.0004186,
+      "loss": 4.769353866577148,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.669629096984863,
+      "eval_runtime": 15.2595,
+      "eval_samples_per_second": 624.332,
+      "eval_steps_per_second": 0.655,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.0004466,
+      "loss": 4.569867324829102,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.00047460000000000004,
+      "loss": 4.389446258544922,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.231207847595215,
+      "eval_runtime": 15.2143,
+      "eval_samples_per_second": 626.188,
+      "eval_steps_per_second": 0.657,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.0005026,
+      "loss": 4.243848419189453,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0005306,
+      "loss": 4.1090648651123045,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.484375,
+      "learning_rate": 0.0005586,
+      "loss": 3.966172790527344,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 3.923773765563965,
+      "eval_runtime": 15.5308,
+      "eval_samples_per_second": 613.426,
+      "eval_steps_per_second": 0.644,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.0005866,
+      "loss": 3.8559597015380858,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.0006146,
+      "loss": 3.7943866729736326,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.6977553367614746,
+      "eval_runtime": 15.2001,
+      "eval_samples_per_second": 626.77,
+      "eval_steps_per_second": 0.658,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0006426,
+      "loss": 3.686443328857422,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0006705999999999999,
+      "loss": 3.6123157501220704,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.0006986,
+      "loss": 3.542654800415039,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.513820171356201,
+      "eval_runtime": 15.2193,
+      "eval_samples_per_second": 625.983,
+      "eval_steps_per_second": 0.657,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0007,
+      "loss": 3.472199249267578,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0007,
+      "loss": 3.396291732788086,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.3581483364105225,
+      "eval_runtime": 15.3044,
+      "eval_samples_per_second": 622.502,
+      "eval_steps_per_second": 0.653,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.0007,
+      "loss": 3.347300720214844,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.2885902404785154,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.25723876953125,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.2229573726654053,
+      "eval_runtime": 15.2967,
+      "eval_samples_per_second": 622.812,
+      "eval_steps_per_second": 0.654,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0007,
+      "loss": 3.1902334213256838,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 3.167861557006836,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.120636463165283,
+      "eval_runtime": 15.3751,
+      "eval_samples_per_second": 619.639,
+      "eval_steps_per_second": 0.65,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.0007,
+      "loss": 3.1089324951171875,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 3.0828268051147463,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 3.0396488189697264,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.0276408195495605,
+      "eval_runtime": 15.2735,
+      "eval_samples_per_second": 623.761,
+      "eval_steps_per_second": 0.655,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 3.0034805297851563,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.0007,
+      "loss": 2.96556396484375,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 2.950009346008301,
+      "eval_runtime": 15.243,
+      "eval_samples_per_second": 625.009,
+      "eval_steps_per_second": 0.656,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.0007,
+      "loss": 2.950779151916504,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.956203079223633,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.9017578125,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 2.8844006061553955,
+      "eval_runtime": 15.2903,
+      "eval_samples_per_second": 623.073,
+      "eval_steps_per_second": 0.654,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0007,
+      "loss": 2.887515640258789,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0007,
+      "loss": 2.8556928634643555,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 2.821610450744629,
+      "eval_runtime": 15.2253,
+      "eval_samples_per_second": 625.733,
+      "eval_steps_per_second": 0.657,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.8130855560302734,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.790329170227051,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0007,
+      "loss": 2.7760162353515625,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 2.7661373615264893,
+      "eval_runtime": 15.2736,
+      "eval_samples_per_second": 623.755,
+      "eval_steps_per_second": 0.655,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.0007,
+      "loss": 2.7462047576904296,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.4140625,
+      "learning_rate": 0.0007,
+      "loss": 2.732274627685547,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 2.713942289352417,
+      "eval_runtime": 15.6062,
+      "eval_samples_per_second": 610.463,
+      "eval_steps_per_second": 0.641,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.0007,
+      "loss": 2.705318069458008,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0007,
+      "loss": 2.6892818450927733,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0007,
+      "loss": 2.687115287780762,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.668647289276123,
+      "eval_runtime": 15.2597,
+      "eval_samples_per_second": 624.326,
+      "eval_steps_per_second": 0.655,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.6533071517944338,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.6390922546386717,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07077856420626896,
+      "eval_loss": 2.6291816234588623,
+      "eval_runtime": 15.2457,
+      "eval_samples_per_second": 624.896,
+      "eval_steps_per_second": 0.656,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.0007,
+      "loss": 2.624461555480957,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.6044427871704103,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.597646713256836,
+      "step": 1100
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "eval_loss": 2.589696168899536,
+      "eval_runtime": 15.3967,
+      "eval_samples_per_second": 618.768,
+      "eval_steps_per_second": 0.649,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.561947250366211,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.390625,
+      "learning_rate": 0.0007,
+      "loss": 2.54862117767334,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07751937984496124,
+      "eval_loss": 2.559185743331909,
+      "eval_runtime": 15.2877,
+      "eval_samples_per_second": 623.181,
+      "eval_steps_per_second": 0.654,
+      "step": 1150
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.0007,
+      "loss": 2.5469717025756835,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0007,
+      "loss": 2.540451240539551,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0007,
+      "loss": 2.53134708404541,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 2.521665096282959,
+      "eval_runtime": 15.2822,
+      "eval_samples_per_second": 623.407,
+      "eval_steps_per_second": 0.654,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0007,
+      "loss": 2.513539123535156,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.5009738922119142,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08426019548365352,
+      "eval_loss": 2.489459753036499,
+      "eval_runtime": 15.2493,
+      "eval_samples_per_second": 624.749,
+      "eval_steps_per_second": 0.656,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.4894077301025392,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.0007,
+      "loss": 2.4697439193725588,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.0007,
+      "loss": 2.461294174194336,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "eval_loss": 2.4627816677093506,
+      "eval_runtime": 15.421,
+      "eval_samples_per_second": 617.795,
+      "eval_steps_per_second": 0.648,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.452680206298828,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0007,
+      "loss": 2.4396684646606444,
+      "step": 1340
+    },
+    {
+      "epoch": 0.0910010111223458,
+      "eval_loss": 2.437220335006714,
+      "eval_runtime": 15.3026,
+      "eval_samples_per_second": 622.576,
+      "eval_steps_per_second": 0.653,
+      "step": 1350
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.0007,
+      "loss": 2.4397760391235352,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.4158536911010744,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.406942367553711,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 2.4096601009368896,
+      "eval_runtime": 15.357,
+      "eval_samples_per_second": 620.368,
+      "eval_steps_per_second": 0.651,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.4161144256591798,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.392984390258789,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09774182676103808,
+      "eval_loss": 2.3937113285064697,
+      "eval_runtime": 15.3555,
+      "eval_samples_per_second": 620.43,
+      "eval_steps_per_second": 0.651,
+      "step": 1450
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0007,
+      "loss": 2.3820331573486326,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.3687986373901366,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0007,
+      "loss": 2.355032730102539,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "eval_loss": 2.3681657314300537,
+      "eval_runtime": 15.3336,
+      "eval_samples_per_second": 621.317,
+      "eval_steps_per_second": 0.652,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10246039770812268,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0007,
+      "loss": 2.3572179794311525,
+      "step": 1520
+    },
+    {
+      "epoch": 0.10380856083586114,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.345840835571289,
+      "step": 1540
+    },
+    {
+      "epoch": 0.10448264239973036,
+      "eval_loss": 2.3475193977355957,
+      "eval_runtime": 15.3453,
+      "eval_samples_per_second": 620.842,
+      "eval_steps_per_second": 0.652,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1051567239635996,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.335231971740723,
+      "step": 1560
+    },
+    {
+      "epoch": 0.10650488709133805,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.3543283462524416,
+      "step": 1580
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.0007,
+      "loss": 2.3237049102783205,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "eval_loss": 2.3377676010131836,
+      "eval_runtime": 15.2858,
+      "eval_samples_per_second": 623.259,
+      "eval_steps_per_second": 0.654,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10920121334681497,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.3196277618408203,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11054937647455342,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.0007,
+      "loss": 2.3308162689208984,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11122345803842265,
+      "eval_loss": 2.313627004623413,
+      "eval_runtime": 15.276,
+      "eval_samples_per_second": 623.659,
+      "eval_steps_per_second": 0.655,
+      "step": 1650
+    },
+    {
+      "epoch": 0.11189753960229187,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.3104904174804686,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11324570273003033,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2955816268920897,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.3044401168823243,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "eval_loss": 2.2976956367492676,
+      "eval_runtime": 15.3404,
+      "eval_samples_per_second": 621.039,
+      "eval_steps_per_second": 0.652,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11594202898550725,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.280081939697266,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1172901921132457,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0007,
+      "loss": 2.269576644897461,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11796427367711493,
+      "eval_loss": 2.28181791305542,
+      "eval_runtime": 15.2137,
+      "eval_samples_per_second": 626.211,
+      "eval_steps_per_second": 0.657,
+      "step": 1750
+    },
+    {
+      "epoch": 0.11863835524098416,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.273081398010254,
+      "step": 1760
+    },
+    {
+      "epoch": 0.11998651836872262,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.2877025604248047,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2828989028930664,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "eval_loss": 2.2671334743499756,
+      "eval_runtime": 15.1615,
+      "eval_samples_per_second": 628.369,
+      "eval_steps_per_second": 0.66,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12268284462419952,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2743486404418944,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0007,
+      "loss": 2.2429489135742187,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12470508931580722,
+      "eval_loss": 2.252441644668579,
+      "eval_runtime": 15.3255,
+      "eval_samples_per_second": 621.643,
+      "eval_steps_per_second": 0.653,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12537917087967643,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0007,
+      "loss": 2.242429733276367,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1267273340074149,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0007,
+      "loss": 2.2348007202148437,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0007,
+      "loss": 2.241002655029297,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "eval_loss": 2.2387285232543945,
+      "eval_runtime": 15.2942,
+      "eval_samples_per_second": 622.915,
+      "eval_steps_per_second": 0.654,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12942366026289182,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.0007,
+      "loss": 2.2172693252563476,
+      "step": 1920
+    },
+    {
+      "epoch": 0.13077182339063026,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.2283090591430663,
+      "step": 1940
+    },
+    {
+      "epoch": 0.13144590495449948,
+      "eval_loss": 2.225919246673584,
+      "eval_runtime": 15.3259,
+      "eval_samples_per_second": 621.627,
+      "eval_steps_per_second": 0.652,
+      "step": 1950
+    },
+    {
+      "epoch": 0.13211998651836873,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0007,
+      "loss": 2.2027830123901366,
+      "step": 1960
+    },
+    {
+      "epoch": 0.13346814964610718,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0007,
+      "loss": 2.224252128601074,
+      "step": 1980
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0007,
+      "loss": 2.1990388870239257,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "eval_loss": 2.215721368789673,
+      "eval_runtime": 15.3022,
+      "eval_samples_per_second": 622.59,
+      "eval_steps_per_second": 0.654,
+      "step": 2000
+    },
+    {
+      "epoch": 0.1361644759015841,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0007,
+      "loss": 2.207795524597168,
+      "step": 2020
+    },
+    {
+      "epoch": 0.13751263902932254,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0007,
+      "loss": 2.1882051467895507,
+      "step": 2040
+    },
+    {
+      "epoch": 0.1381867205931918,
+      "eval_loss": 2.2054405212402344,
+      "eval_runtime": 15.4183,
+      "eval_samples_per_second": 617.904,
+      "eval_steps_per_second": 0.649,
+      "step": 2050
+    },
+    {
+      "epoch": 0.138860802157061,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0007,
+      "loss": 2.206527900695801,
+      "step": 2060
+    },
+    {
+      "epoch": 0.14020896528479945,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0007,
+      "loss": 2.1837987899780273,
+      "step": 2080
+    },
+    {
+      "epoch": 0.14155712841253792,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0007,
+      "loss": 2.1744739532470705,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14155712841253792,
+      "eval_loss": 2.1948788166046143,
+      "eval_runtime": 15.1797,
+      "eval_samples_per_second": 627.615,
+      "eval_steps_per_second": 0.659,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14290529154027637,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0007,
+      "loss": 2.182137298583984,
+      "step": 2120
+    },
+    {
+      "epoch": 0.14425345466801484,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0007,
+      "loss": 2.1851844787597656,
+      "step": 2140
+    },
+    {
+      "epoch": 0.14492753623188406,
+      "eval_loss": 2.1849257946014404,
+      "eval_runtime": 15.2467,
+      "eval_samples_per_second": 624.858,
+      "eval_steps_per_second": 0.656,
+      "step": 2150
+    },
+    {
+      "epoch": 0.14560161779575329,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0007,
+      "loss": 2.1681228637695313,
+      "step": 2160
+    },
+    {
+      "epoch": 0.14694978092349173,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.0007,
+      "loss": 2.1777414321899413,
+      "step": 2180
+    },
+    {
+      "epoch": 0.1482979440512302,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0007,
+      "loss": 2.1520442962646484,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1482979440512302,
+      "eval_loss": 2.1730399131774902,
+      "eval_runtime": 15.3329,
+      "eval_samples_per_second": 621.344,
+      "eval_steps_per_second": 0.652,
+      "step": 2200
+    },
+    {
+      "epoch": 0.14964610717896865,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0007,
+      "loss": 2.159272384643555,
+      "step": 2220
+    },
+    {
+      "epoch": 0.15099427030670712,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0007,
+      "loss": 2.1691991806030275,
+      "step": 2240
+    },
+    {
+      "epoch": 0.15166835187057634,
+      "eval_loss": 2.1636791229248047,
+      "eval_runtime": 15.4687,
+      "eval_samples_per_second": 615.888,
+      "eval_steps_per_second": 0.646,
+      "step": 2250
+    },
+    {
+      "epoch": 0.15234243343444556,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0007,
+      "loss": 2.147579383850098,
+      "step": 2260
+    },
+    {
+      "epoch": 0.15369059656218403,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.0007,
+      "loss": 2.1597997665405275,
+      "step": 2280
+    },
+    {
+      "epoch": 0.15503875968992248,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0007,
+      "loss": 2.1413923263549806,
+      "step": 2300
+    },
+    {
+      "epoch": 0.15503875968992248,
+      "eval_loss": 2.1545023918151855,
+      "eval_runtime": 15.4261,
+      "eval_samples_per_second": 617.588,
+      "eval_steps_per_second": 0.648,
+      "step": 2300
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7420452746035200.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-100L_run/checkpoint-2300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd90512e56f813612dc668676d6f8cab72da110a42f23c61cafebf91064625a1
+size 4920

zain/Activation/out/glu-silu-100L_run/training_log.jsonl

@@ -128,3 +128,38 @@
 {"step": 1800, "epoch": 0.12133468149646107, "timestamp": 1786571655.0203607, "eval_loss": 2.2671334743499756, "eval_runtime": 15.1615, "eval_samples_per_second": 628.369, "eval_steps_per_second": 0.66, "train/total_time_seconds": 425.08155478164554, "train/time_per_step_avg": 0.2376301397010684, "train/epoch_time_elapsed": 1069.6787635795772, "train/estimated_remaining_minutes": 2.7551582254365914}
 {"step": 1820, "epoch": 0.12268284462419952, "timestamp": 1786571660.9795668, "loss": 2.2743486404418944, "grad_norm": 0.388671875, "learning_rate": 0.0007, "train/total_time_seconds": 429.7808753103018, "train/time_per_step_avg": 0.23479793954640626, "train/epoch_time_elapsed": 1075.637973241508, "train/estimated_remaining_minutes": 2.6762911649359453}
 {"step": 1840, "epoch": 0.12403100775193798, "timestamp": 1786571666.6678298, "loss": 2.2429489135742187, "grad_norm": 0.3671875, "learning_rate": 0.0007, "train/total_time_seconds": 434.4591994471848, "train/time_per_step_avg": 0.23448411744087935, "train/epoch_time_elapsed": 1081.3262354955077, "train/estimated_remaining_minutes": 2.5973104314777355}
+{"step": 1850, "epoch": 0.12470508931580722, "timestamp": 1786571684.8217556, "eval_loss": 2.252441644668579, "eval_runtime": 15.3255, "eval_samples_per_second": 621.643, "eval_steps_per_second": 0.653, "train/total_time_seconds": 436.7967942431569, "train/time_per_step_avg": 0.23446216210722923, "train/epoch_time_elapsed": 1099.4801595695317, "train/estimated_remaining_minutes": 2.5578190653878554}
+{"step": 1860, "epoch": 0.12537917087967643, "timestamp": 1786571687.7066696, "loss": 2.242429733276367, "grad_norm": 0.33984375, "learning_rate": 0.0007, "train/total_time_seconds": 439.1811627559364, "train/time_per_step_avg": 0.23492698848247529, "train/epoch_time_elapsed": 1102.3650759272277, "train/estimated_remaining_minutes": 2.518601650213255}
+{"step": 1880, "epoch": 0.1267273340074149, "timestamp": 1786571693.6856463, "loss": 2.2348007202148437, "grad_norm": 0.353515625, "learning_rate": 0.0007, "train/total_time_seconds": 444.09829803183675, "train/time_per_step_avg": 0.23730077136307956, "train/epoch_time_elapsed": 1108.3440522402525, "train/estimated_remaining_minutes": 2.4409658225154147}
+{"step": 1900, "epoch": 0.12807549713515334, "timestamp": 1786571699.4580579, "loss": 2.241002655029297, "grad_norm": 0.330078125, "learning_rate": 0.0007, "train/total_time_seconds": 448.87546668201685, "train/time_per_step_avg": 0.23793911900371312, "train/epoch_time_elapsed": 1114.1164647303522, "train/estimated_remaining_minutes": 2.3625024562211414}
+{"step": 1900, "epoch": 0.12807549713515334, "timestamp": 1786571714.7538586, "eval_loss": 2.2387285232543945, "eval_runtime": 15.2942, "eval_samples_per_second": 622.915, "eval_steps_per_second": 0.654, "train/total_time_seconds": 448.87546668201685, "train/time_per_step_avg": 0.23793911900371312, "train/epoch_time_elapsed": 1129.412262685597, "train/estimated_remaining_minutes": 2.3625024562211414}
+{"step": 1920, "epoch": 0.12942366026289182, "timestamp": 1786571720.822342, "loss": 2.2172693252563476, "grad_norm": 0.34765625, "learning_rate": 0.0007, "train/total_time_seconds": 453.64645610004663, "train/time_per_step_avg": 0.23865580789744853, "train/epoch_time_elapsed": 1135.4807489812374, "train/estimated_remaining_minutes": 2.2839838935592622}
+{"step": 1940, "epoch": 0.13077182339063026, "timestamp": 1786571726.8973782, "loss": 2.2283090591430663, "grad_norm": 0.357421875, "learning_rate": 0.0007, "train/total_time_seconds": 458.68312226608396, "train/time_per_step_avg": 0.24223922818899155, "train/epoch_time_elapsed": 1141.5557847991586, "train/estimated_remaining_minutes": 2.2067229249914693}
+{"step": 1950, "epoch": 0.13144590495449948, "timestamp": 1786571745.142048, "eval_loss": 2.225919246673584, "eval_runtime": 15.3259, "eval_samples_per_second": 621.627, "eval_steps_per_second": 0.652, "train/total_time_seconds": 461.0843236260116, "train/time_per_step_avg": 0.242875293828547, "train/epoch_time_elapsed": 1159.800451811403, "train/estimated_remaining_minutes": 2.167490410207747}
+{"step": 1960, "epoch": 0.13211998651836873, "timestamp": 1786571748.0502758, "loss": 2.2027830123901366, "grad_norm": 0.333984375, "learning_rate": 0.0007, "train/total_time_seconds": 463.4878015592694, "train/time_per_step_avg": 0.24306638803333044, "train/epoch_time_elapsed": 1162.7086823098361, "train/estimated_remaining_minutes": 2.1282603132823597}
+{"step": 1980, "epoch": 0.13346814964610718, "timestamp": 1786571753.852559, "loss": 2.224252128601074, "grad_norm": 0.359375, "learning_rate": 0.0007, "train/total_time_seconds": 468.2867293320596, "train/time_per_step_avg": 0.24188431300222873, "train/epoch_time_elapsed": 1168.510965935886, "train/estimated_remaining_minutes": 2.049739892699251}
+{"step": 2000, "epoch": 0.13481631277384565, "timestamp": 1786571759.6433632, "loss": 2.1990388870239257, "grad_norm": 0.35546875, "learning_rate": 0.0007, "train/total_time_seconds": 473.06068509072065, "train/time_per_step_avg": 0.24185218408703804, "train/epoch_time_elapsed": 1174.3017696514726, "train/estimated_remaining_minutes": 1.9710861878780028}
+{"step": 2000, "epoch": 0.13481631277384565, "timestamp": 1786571774.9470952, "eval_loss": 2.215721368789673, "eval_runtime": 15.3022, "eval_samples_per_second": 622.59, "eval_steps_per_second": 0.654, "train/total_time_seconds": 473.06068509072065, "train/time_per_step_avg": 0.24185218408703804, "train/epoch_time_elapsed": 1189.605499446392, "train/estimated_remaining_minutes": 1.9710861878780028}
+{"step": 2020, "epoch": 0.1361644759015841, "timestamp": 1786571781.0423207, "loss": 2.207795524597168, "grad_norm": 0.345703125, "learning_rate": 0.0007, "train/total_time_seconds": 477.86794033646584, "train/time_per_step_avg": 0.242214842364192, "train/epoch_time_elapsed": 1195.70072709769, "train/estimated_remaining_minutes": 1.892546298362241}
+{"step": 2040, "epoch": 0.13751263902932254, "timestamp": 1786571786.827905, "loss": 2.1882051467895507, "grad_norm": 0.396484375, "learning_rate": 0.0007, "train/total_time_seconds": 482.6602343544364, "train/time_per_step_avg": 0.23977112088352442, "train/epoch_time_elapsed": 1201.4863113425672, "train/estimated_remaining_minutes": 1.8139191813973916}
+{"step": 2050, "epoch": 0.1381867205931918, "timestamp": 1786571805.1550922, "eval_loss": 2.2054405212402344, "eval_runtime": 15.4183, "eval_samples_per_second": 617.904, "eval_steps_per_second": 0.649, "train/total_time_seconds": 485.07023257762194, "train/time_per_step_avg": 0.23985908951610327, "train/epoch_time_elapsed": 1219.813495900482, "train/estimated_remaining_minutes": 1.7746471923571534}
+{"step": 2060, "epoch": 0.138860802157061, "timestamp": 1786571807.990259, "loss": 2.206527900695801, "grad_norm": 0.365234375, "learning_rate": 0.0007, "train/total_time_seconds": 487.4215290360153, "train/time_per_step_avg": 0.23933727476745845, "train/epoch_time_elapsed": 1222.64866348356, "train/estimated_remaining_minutes": 1.7351575467301514}
+{"step": 2080, "epoch": 0.14020896528479945, "timestamp": 1786571813.6674707, "loss": 2.1837987899780273, "grad_norm": 0.361328125, "learning_rate": 0.0007, "train/total_time_seconds": 492.1069700829685, "train/time_per_step_avg": 0.23820240750908853, "train/epoch_time_elapsed": 1228.3258763849735, "train/estimated_remaining_minutes": 1.6561292262407592}
+{"step": 2100, "epoch": 0.14155712841253792, "timestamp": 1786571819.4294372, "loss": 2.1744739532470705, "grad_norm": 0.33203125, "learning_rate": 0.0007, "train/total_time_seconds": 496.86956125870347, "train/time_per_step_avg": 0.23808876167982818, "train/epoch_time_elapsed": 1234.0878428667784, "train/estimated_remaining_minutes": 1.5773636865355665}
+{"step": 2100, "epoch": 0.14155712841253792, "timestamp": 1786571834.610596, "eval_loss": 2.1948788166046143, "eval_runtime": 15.1797, "eval_samples_per_second": 627.615, "eval_steps_per_second": 0.659, "train/total_time_seconds": 496.86956125870347, "train/time_per_step_avg": 0.23808876167982818, "train/epoch_time_elapsed": 1249.2689991667867, "train/estimated_remaining_minutes": 1.5773636865355665}
+{"step": 2120, "epoch": 0.14290529154027637, "timestamp": 1786571840.5458899, "loss": 2.182137298583984, "grad_norm": 0.361328125, "learning_rate": 0.0007, "train/total_time_seconds": 501.5555355772376, "train/time_per_step_avg": 0.2368759524077177, "train/epoch_time_elapsed": 1255.204294770956, "train/estimated_remaining_minutes": 1.4983577320703638}
+{"step": 2140, "epoch": 0.14425345466801484, "timestamp": 1786571846.5151026, "loss": 2.1851844787597656, "grad_norm": 0.337890625, "learning_rate": 0.0007, "train/total_time_seconds": 506.5370416678488, "train/time_per_step_avg": 0.2387680731341243, "train/epoch_time_elapsed": 1261.1735085994005, "train/estimated_remaining_minutes": 1.4201973130874268}
+{"step": 2150, "epoch": 0.14492753623188406, "timestamp": 1786571864.6563807, "eval_loss": 2.1849257946014404, "eval_runtime": 15.2467, "eval_samples_per_second": 624.858, "eval_steps_per_second": 0.656, "train/total_time_seconds": 508.9251952022314, "train/time_per_step_avg": 0.23854962624609471, "train/epoch_time_elapsed": 1279.314786080271, "train/estimated_remaining_minutes": 1.3808047931843488}
+{"step": 2160, "epoch": 0.14560161779575329, "timestamp": 1786571867.583958, "loss": 2.1681228637695313, "grad_norm": 0.3515625, "learning_rate": 0.0007, "train/total_time_seconds": 511.35102678462863, "train/time_per_step_avg": 0.23929497748613357, "train/epoch_time_elapsed": 1282.2423641048372, "train/estimated_remaining_minutes": 1.3415073233547357}
+{"step": 2180, "epoch": 0.14694978092349173, "timestamp": 1786571873.3709059, "loss": 2.1777414321899413, "grad_norm": 0.34375, "learning_rate": 0.0007, "train/total_time_seconds": 516.1408885456622, "train/time_per_step_avg": 0.2403391846269369, "train/epoch_time_elapsed": 1288.0293121039867, "train/estimated_remaining_minutes": 1.2627300025581951}
+{"step": 2200, "epoch": 0.1482979440512302, "timestamp": 1786571879.287127, "loss": 2.1520442962646484, "grad_norm": 0.33203125, "learning_rate": 0.0007, "train/total_time_seconds": 521.0086051784456, "train/time_per_step_avg": 0.24139043919742106, "train/epoch_time_elapsed": 1293.9455332793295, "train/estimated_remaining_minutes": 1.1841104663146491}
+{"step": 2200, "epoch": 0.1482979440512302, "timestamp": 1786571894.6215513, "eval_loss": 2.1730399131774902, "eval_runtime": 15.3329, "eval_samples_per_second": 621.344, "eval_steps_per_second": 0.652, "train/total_time_seconds": 521.0086051784456, "train/time_per_step_avg": 0.24139043919742106, "train/epoch_time_elapsed": 1309.279956497252, "train/estimated_remaining_minutes": 1.1841104663146491}
+{"step": 2220, "epoch": 0.14964610717896865, "timestamp": 1786571900.691944, "loss": 2.159272384643555, "grad_norm": 0.333984375, "learning_rate": 0.0007, "train/total_time_seconds": 525.7687088884413, "train/time_per_step_avg": 0.24213173311203717, "train/epoch_time_elapsed": 1315.3503500632942, "train/estimated_remaining_minutes": 1.1052195081739005}
+{"step": 2240, "epoch": 0.15099427030670712, "timestamp": 1786571906.484491, "loss": 2.1691991806030275, "grad_norm": 0.357421875, "learning_rate": 0.0007, "train/total_time_seconds": 530.54325793311, "train/time_per_step_avg": 0.24006216265261174, "train/epoch_time_elapsed": 1321.142897989601, "train/estimated_remaining_minutes": 1.026348564453933}
+{"step": 2250, "epoch": 0.15166835187057634, "timestamp": 1786571924.8611524, "eval_loss": 2.1636791229248047, "eval_runtime": 15.4687, "eval_samples_per_second": 615.888, "eval_steps_per_second": 0.646, "train/total_time_seconds": 532.9512122161686, "train/time_per_step_avg": 0.24026017013937234, "train/epoch_time_elapsed": 1339.5195576660335, "train/estimated_remaining_minutes": 0.9869466892892012}
+{"step": 2260, "epoch": 0.15234243343444556, "timestamp": 1786571927.73209, "loss": 2.147579383850098, "grad_norm": 0.36328125, "learning_rate": 0.0007, "train/total_time_seconds": 535.328151807189, "train/time_per_step_avg": 0.23977125022560358, "train/epoch_time_elapsed": 1342.3904966041446, "train/estimated_remaining_minutes": 0.9474834545259984}
+{"step": 2280, "epoch": 0.15369059656218403, "timestamp": 1786571933.486512, "loss": 2.1597997665405275, "grad_norm": 0.34765625, "learning_rate": 0.0007, "train/total_time_seconds": 540.0898813083768, "train/time_per_step_avg": 0.23948992762714624, "train/epoch_time_elapsed": 1348.144918717444, "train/estimated_remaining_minutes": 0.8685655985953428}
+{"step": 2300, "epoch": 0.15503875968992248, "timestamp": 1786571939.3196301, "loss": 2.1413923263549806, "grad_norm": 0.333984375, "learning_rate": 0.0007, "train/total_time_seconds": 544.8871062174439, "train/time_per_step_avg": 0.23878501038998365, "train/epoch_time_elapsed": 1353.9780368097126, "train/estimated_remaining_minutes": 0.7896914582861506}
+{"step": 2300, "epoch": 0.15503875968992248, "timestamp": 1786571954.7473707, "eval_loss": 2.1545023918151855, "eval_runtime": 15.4261, "eval_samples_per_second": 617.588, "eval_steps_per_second": 0.648, "train/total_time_seconds": 544.8871062174439, "train/time_per_step_avg": 0.23878501038998365, "train/epoch_time_elapsed": 1369.4057754948735, "train/estimated_remaining_minutes": 0.7896914582861506}
+{"step": 2320, "epoch": 0.15638692281766095, "timestamp": 1786571960.8122804, "loss": 2.1491439819335936, "grad_norm": 0.33203125, "learning_rate": 0.0007, "train/total_time_seconds": 549.6645726785064, "train/time_per_step_avg": 0.2389586379006505, "train/epoch_time_elapsed": 1375.4706870652735, "train/estimated_remaining_minutes": 0.7107731543256548}
+{"step": 2340, "epoch": 0.1577350859453994, "timestamp": 1786571966.612273, "loss": 2.137864112854004, "grad_norm": 0.33203125, "learning_rate": 0.0007, "train/total_time_seconds": 554.4445280842483, "train/time_per_step_avg": 0.23901270151138307, "train/epoch_time_elapsed": 1381.2706793919206, "train/estimated_remaining_minutes": 0.6318456160504254}

zain/Activation/wandb/debug-internal.log

@@ -151,3 +151,43 @@
 {"time":"2026-08-12T21:54:10.755710662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T21:54:25.573166176Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":125,"history_lines":2,"events_offset":142,"events_lines":2,"console_offset":206,"console_lines":1}
 {"time":"2026-08-12T21:54:25.691084195Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:54:40.57284682Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":127,"history_lines":1,"events_offset":144,"events_lines":2,"console_offset":209,"console_lines":13}
+{"time":"2026-08-12T21:54:40.69762128Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:54:55.572554983Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":128,"history_lines":3,"events_offset":146,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-08-12T21:54:55.679616542Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:10.572913717Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":131,"history_lines":1,"events_offset":148,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-08-12T21:55:10.727795537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:25.572468618Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":132,"history_lines":2,"events_offset":150,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-08-12T21:55:25.687366993Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:40.572815743Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":134,"history_lines":1,"events_offset":152,"events_lines":2,"console_offset":221,"console_lines":13}
+{"time":"2026-08-12T21:55:40.690219687Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:55.572891208Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":135,"history_lines":3,"events_offset":154,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-08-12T21:55:55.689879782Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:10.572897097Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":138,"history_lines":1,"events_offset":156,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-08-12T21:56:10.729258272Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:25.572763312Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":139,"history_lines":2,"events_offset":158,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-08-12T21:56:25.687384946Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:40.572838717Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":141,"history_lines":1,"events_offset":160,"events_lines":2,"console_offset":233,"console_lines":13}
+{"time":"2026-08-12T21:56:40.707823075Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:55.572855246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":142,"history_lines":3,"events_offset":162,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-08-12T21:56:55.725829861Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:10.572920474Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":145,"history_lines":1,"events_offset":164,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-08-12T21:57:10.718947165Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:25.572645607Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":146,"history_lines":2,"events_offset":166,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-08-12T21:57:25.691229906Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:40.572409641Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":148,"history_lines":1,"events_offset":168,"events_lines":2,"console_offset":245,"console_lines":13}
+{"time":"2026-08-12T21:57:40.68755905Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:55.573295924Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":149,"history_lines":3,"events_offset":170,"events_lines":2,"console_offset":254,"console_lines":1}
+{"time":"2026-08-12T21:57:55.6967349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:10.573041844Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":152,"history_lines":1,"events_offset":172,"events_lines":2,"console_offset":254,"console_lines":1}
+{"time":"2026-08-12T21:58:10.728134421Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:25.573293709Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":153,"history_lines":2,"events_offset":174,"events_lines":2,"console_offset":254,"console_lines":1}
+{"time":"2026-08-12T21:58:25.70203417Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:40.572910895Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":155,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":257,"console_lines":13}
+{"time":"2026-08-12T21:58:40.702594215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:55.573258933Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":156,"history_lines":3,"events_offset":178,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-08-12T21:58:55.69998018Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:59:10.572808812Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":159,"history_lines":1,"events_offset":180,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-08-12T21:59:10.716244539Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:59:25.572810198Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":160,"history_lines":2,"events_offset":182,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-08-12T21:59:25.68063981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_213624-1sryr5l2/files/output.log

@@ -216,5 +216,66 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 12.
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 19.28it/s]
- 73%|███████▎  | 1836/2500 [18:01<03:10,  3.49it/s], ?it/s]
+ 76%|███████▌  | 1900/2500 [18:50<02:54,  3.44i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.274', 'grad_norm': '0.3887', 'learning_rate': '0.0007', 'epoch': '0.1227', 'train/total_time_seconds': '429.8', 'train/time_per_step_avg': '0.2348', 'train/epoch_time_elapsed': '1076', 'train/estimated_remaining_minutes': '2.676'}
+{'loss': '2.243', 'grad_norm': '0.3672', 'learning_rate': '0.0007', 'epoch': '0.124', 'train/total_time_seconds': '434.5', 'train/time_per_step_avg': '0.2345', 'train/epoch_time_elapsed': '1081', 'train/estimated_remaining_minutes': '2.597'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.252', 'eval_runtime': '15.33', 'eval_samples_per_second': '621.6', 'eval_steps_per_second': '0.653', 'epoch': '0.1247', 'train/total_time_seconds': '436.8', 'train/time_per_step_avg': '0.2345', 'train/epoch_time_elapsed': '1099', 'train/estimated_remaining_minutes': '2.558'}
+{'loss': '2.242', 'grad_norm': '0.3398', 'learning_rate': '0.0007', 'epoch': '0.1254', 'train/total_time_seconds': '439.2', 'train/time_per_step_avg': '0.2349', 'train/epoch_time_elapsed': '1102', 'train/estimated_remaining_minutes': '2.519'}
+{'loss': '2.235', 'grad_norm': '0.3535', 'learning_rate': '0.0007', 'epoch': '0.1267', 'train/total_time_seconds': '444.1', 'train/time_per_step_avg': '0.2373', 'train/epoch_time_elapsed': '1108', 'train/estimated_remaining_minutes': '2.441'}
+{'loss': '2.241', 'grad_norm': '0.3301', 'learning_rate': '0.0007', 'epoch': '0.1281', 'train/total_time_seconds': '448.9', 'train/time_per_step_avg': '0.2379', 'train/epoch_time_elapsed': '1114', 'train/estimated_remaining_minutes': '2.363'}
+{'eval_loss': '2.239', 'eval_runtime': '15.29', 'eval_samples_per_second': '622.9', 'eval_steps_per_second': '0.654', 'epoch': '0.1281', 'train/total_time_seconds': '448.9', 'train/time_per_step_avg': '0.2379', 'train/epoch_time_elapsed': '1129', 'train/estimated_remaining_minutes': '2.363'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.22it/s]
+ 80%|████████  | 2000/2500 [19:50<02:24,  3.45i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.217', 'grad_norm': '0.3477', 'learning_rate': '0.0007', 'epoch': '0.1294', 'train/total_time_seconds': '453.6', 'train/time_per_step_avg': '0.2387', 'train/epoch_time_elapsed': '1135', 'train/estimated_remaining_minutes': '2.284'}
+{'loss': '2.228', 'grad_norm': '0.3574', 'learning_rate': '0.0007', 'epoch': '0.1308', 'train/total_time_seconds': '458.7', 'train/time_per_step_avg': '0.2422', 'train/epoch_time_elapsed': '1142', 'train/estimated_remaining_minutes': '2.207'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.226', 'eval_runtime': '15.33', 'eval_samples_per_second': '621.6', 'eval_steps_per_second': '0.652', 'epoch': '0.1314', 'train/total_time_seconds': '461.1', 'train/time_per_step_avg': '0.2429', 'train/epoch_time_elapsed': '1160', 'train/estimated_remaining_minutes': '2.167'}
+{'loss': '2.203', 'grad_norm': '0.334', 'learning_rate': '0.0007', 'epoch': '0.1321', 'train/total_time_seconds': '463.5', 'train/time_per_step_avg': '0.2431', 'train/epoch_time_elapsed': '1163', 'train/estimated_remaining_minutes': '2.128'}
+{'loss': '2.224', 'grad_norm': '0.3594', 'learning_rate': '0.0007', 'epoch': '0.1335', 'train/total_time_seconds': '468.3', 'train/time_per_step_avg': '0.2419', 'train/epoch_time_elapsed': '1169', 'train/estimated_remaining_minutes': '2.05'}
+{'loss': '2.199', 'grad_norm': '0.3555', 'learning_rate': '0.0007', 'epoch': '0.1348', 'train/total_time_seconds': '473.1', 'train/time_per_step_avg': '0.2419', 'train/epoch_time_elapsed': '1174', 'train/estimated_remaining_minutes': '1.971'}
+{'eval_loss': '2.216', 'eval_runtime': '15.3', 'eval_samples_per_second': '622.6', 'eval_steps_per_second': '0.654', 'epoch': '0.1348', 'train/total_time_seconds': '473.1', 'train/time_per_step_avg': '0.2419', 'train/epoch_time_elapsed': '1190', 'train/estimated_remaining_minutes': '1.971'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.91it/s]
+ 84%|████████▍ | 2100/2500 [20:50<01:53,  3.53i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.208', 'grad_norm': '0.3457', 'learning_rate': '0.0007', 'epoch': '0.1362', 'train/total_time_seconds': '477.9', 'train/time_per_step_avg': '0.2422', 'train/epoch_time_elapsed': '1196', 'train/estimated_remaining_minutes': '1.893'}
+{'loss': '2.188', 'grad_norm': '0.3965', 'learning_rate': '0.0007', 'epoch': '0.1375', 'train/total_time_seconds': '482.7', 'train/time_per_step_avg': '0.2398', 'train/epoch_time_elapsed': '1201', 'train/estimated_remaining_minutes': '1.814'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.205', 'eval_runtime': '15.42', 'eval_samples_per_second': '617.9', 'eval_steps_per_second': '0.649', 'epoch': '0.1382', 'train/total_time_seconds': '485.1', 'train/time_per_step_avg': '0.2399', 'train/epoch_time_elapsed': '1220', 'train/estimated_remaining_minutes': '1.775'}
+{'loss': '2.207', 'grad_norm': '0.3652', 'learning_rate': '0.0007', 'epoch': '0.1389', 'train/total_time_seconds': '487.4', 'train/time_per_step_avg': '0.2393', 'train/epoch_time_elapsed': '1223', 'train/estimated_remaining_minutes': '1.735'}
+{'loss': '2.184', 'grad_norm': '0.3613', 'learning_rate': '0.0007', 'epoch': '0.1402', 'train/total_time_seconds': '492.1', 'train/time_per_step_avg': '0.2382', 'train/epoch_time_elapsed': '1228', 'train/estimated_remaining_minutes': '1.656'}
+{'loss': '2.174', 'grad_norm': '0.332', 'learning_rate': '0.0007', 'epoch': '0.1416', 'train/total_time_seconds': '496.9', 'train/time_per_step_avg': '0.2381', 'train/epoch_time_elapsed': '1234', 'train/estimated_remaining_minutes': '1.577'}
+{'eval_loss': '2.195', 'eval_runtime': '15.18', 'eval_samples_per_second': '627.6', 'eval_steps_per_second': '0.659', 'epoch': '0.1416', 'train/total_time_seconds': '496.9', 'train/time_per_step_avg': '0.2381', 'train/epoch_time_elapsed': '1249', 'train/estimated_remaining_minutes': '1.577'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.89it/s]
+ 88%|████████▊ | 2200/2500 [21:50<01:34,  3.18i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.182', 'grad_norm': '0.3613', 'learning_rate': '0.0007', 'epoch': '0.1429', 'train/total_time_seconds': '501.6', 'train/time_per_step_avg': '0.2369', 'train/epoch_time_elapsed': '1255', 'train/estimated_remaining_minutes': '1.498'}
+{'loss': '2.185', 'grad_norm': '0.3379', 'learning_rate': '0.0007', 'epoch': '0.1443', 'train/total_time_seconds': '506.5', 'train/time_per_step_avg': '0.2388', 'train/epoch_time_elapsed': '1261', 'train/estimated_remaining_minutes': '1.42'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.185', 'eval_runtime': '15.25', 'eval_samples_per_second': '624.9', 'eval_steps_per_second': '0.656', 'epoch': '0.1449', 'train/total_time_seconds': '508.9', 'train/time_per_step_avg': '0.2385', 'train/epoch_time_elapsed': '1279', 'train/estimated_remaining_minutes': '1.381'}
+{'loss': '2.168', 'grad_norm': '0.3516', 'learning_rate': '0.0007', 'epoch': '0.1456', 'train/total_time_seconds': '511.4', 'train/time_per_step_avg': '0.2393', 'train/epoch_time_elapsed': '1282', 'train/estimated_remaining_minutes': '1.342'}
+{'loss': '2.178', 'grad_norm': '0.3438', 'learning_rate': '0.0007', 'epoch': '0.1469', 'train/total_time_seconds': '516.1', 'train/time_per_step_avg': '0.2403', 'train/epoch_time_elapsed': '1288', 'train/estimated_remaining_minutes': '1.263'}
+{'loss': '2.152', 'grad_norm': '0.332', 'learning_rate': '0.0007', 'epoch': '0.1483', 'train/total_time_seconds': '521', 'train/time_per_step_avg': '0.2414', 'train/epoch_time_elapsed': '1294', 'train/estimated_remaining_minutes': '1.184'}
+{'eval_loss': '2.173', 'eval_runtime': '15.33', 'eval_samples_per_second': '621.3', 'eval_steps_per_second': '0.652', 'epoch': '0.1483', 'train/total_time_seconds': '521', 'train/time_per_step_avg': '0.2414', 'train/epoch_time_elapsed': '1309', 'train/estimated_remaining_minutes': '1.184'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 10.97it/s]
+ 92%|█████████▏| 2300/2500 [22:50<00:58,  3.44i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.159', 'grad_norm': '0.334', 'learning_rate': '0.0007', 'epoch': '0.1496', 'train/total_time_seconds': '525.8', 'train/time_per_step_avg': '0.2421', 'train/epoch_time_elapsed': '1315', 'train/estimated_remaining_minutes': '1.105'}
+{'loss': '2.169', 'grad_norm': '0.3574', 'learning_rate': '0.0007', 'epoch': '0.151', 'train/total_time_seconds': '530.5', 'train/time_per_step_avg': '0.2401', 'train/epoch_time_elapsed': '1321', 'train/estimated_remaining_minutes': '1.026'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.164', 'eval_runtime': '15.47', 'eval_samples_per_second': '615.9', 'eval_steps_per_second': '0.646', 'epoch': '0.1517', 'train/total_time_seconds': '533', 'train/time_per_step_avg': '0.2403', 'train/epoch_time_elapsed': '1340', 'train/estimated_remaining_minutes': '0.9869'}
+{'loss': '2.148', 'grad_norm': '0.3633', 'learning_rate': '0.0007', 'epoch': '0.1523', 'train/total_time_seconds': '535.3', 'train/time_per_step_avg': '0.2398', 'train/epoch_time_elapsed': '1342', 'train/estimated_remaining_minutes': '0.9475'}
+{'loss': '2.16', 'grad_norm': '0.3477', 'learning_rate': '0.0007', 'epoch': '0.1537', 'train/total_time_seconds': '540.1', 'train/time_per_step_avg': '0.2395', 'train/epoch_time_elapsed': '1348', 'train/estimated_remaining_minutes': '0.8686'}
+{'loss': '2.141', 'grad_norm': '0.334', 'learning_rate': '0.0007', 'epoch': '0.155', 'train/total_time_seconds': '544.9', 'train/time_per_step_avg': '0.2388', 'train/epoch_time_elapsed': '1354', 'train/estimated_remaining_minutes': '0.7897'}
+{'eval_loss': '2.155', 'eval_runtime': '15.43', 'eval_samples_per_second': '617.6', 'eval_steps_per_second': '0.648', 'epoch': '0.155', 'train/total_time_seconds': '544.9', 'train/time_per_step_avg': '0.2388', 'train/epoch_time_elapsed': '1369', 'train/estimated_remaining_minutes': '0.7897'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.80it/s]
+ 94%|█████████▍| 2350/2500 [23:05<00:43,  3.47it/s], ?it/s]
+{'loss': '2.149', 'grad_norm': '0.332', 'learning_rate': '0.0007', 'epoch': '0.1564', 'train/total_time_seconds': '549.7', 'train/time_per_step_avg': '0.239', 'train/epoch_time_elapsed': '1375', 'train/estimated_remaining_minutes': '0.7108'}
+{'loss': '2.138', 'grad_norm': '0.332', 'learning_rate': '0.0007', 'epoch': '0.1577', 'train/total_time_seconds': '554.4', 'train/time_per_step_avg': '0.239', 'train/epoch_time_elapsed': '1381', 'train/estimated_remaining_minutes': '0.6318'}

zain/Activation/wandb/run-20260812_213624-1sryr5l2/logs/debug-internal.log

@@ -151,3 +151,43 @@
 {"time":"2026-08-12T21:54:10.755710662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T21:54:25.573166176Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":125,"history_lines":2,"events_offset":142,"events_lines":2,"console_offset":206,"console_lines":1}
 {"time":"2026-08-12T21:54:25.691084195Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:54:40.57284682Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":127,"history_lines":1,"events_offset":144,"events_lines":2,"console_offset":209,"console_lines":13}
+{"time":"2026-08-12T21:54:40.69762128Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:54:55.572554983Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":128,"history_lines":3,"events_offset":146,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-08-12T21:54:55.679616542Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:10.572913717Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":131,"history_lines":1,"events_offset":148,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-08-12T21:55:10.727795537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:25.572468618Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":132,"history_lines":2,"events_offset":150,"events_lines":2,"console_offset":218,"console_lines":1}
+{"time":"2026-08-12T21:55:25.687366993Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:40.572815743Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":134,"history_lines":1,"events_offset":152,"events_lines":2,"console_offset":221,"console_lines":13}
+{"time":"2026-08-12T21:55:40.690219687Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:55:55.572891208Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":135,"history_lines":3,"events_offset":154,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-08-12T21:55:55.689879782Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:10.572897097Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":138,"history_lines":1,"events_offset":156,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-08-12T21:56:10.729258272Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:25.572763312Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":139,"history_lines":2,"events_offset":158,"events_lines":2,"console_offset":230,"console_lines":1}
+{"time":"2026-08-12T21:56:25.687384946Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:40.572838717Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":141,"history_lines":1,"events_offset":160,"events_lines":2,"console_offset":233,"console_lines":13}
+{"time":"2026-08-12T21:56:40.707823075Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:56:55.572855246Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":142,"history_lines":3,"events_offset":162,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-08-12T21:56:55.725829861Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:10.572920474Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":145,"history_lines":1,"events_offset":164,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-08-12T21:57:10.718947165Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:25.572645607Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":146,"history_lines":2,"events_offset":166,"events_lines":2,"console_offset":242,"console_lines":1}
+{"time":"2026-08-12T21:57:25.691229906Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:40.572409641Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":148,"history_lines":1,"events_offset":168,"events_lines":2,"console_offset":245,"console_lines":13}
+{"time":"2026-08-12T21:57:40.68755905Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:57:55.573295924Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":149,"history_lines":3,"events_offset":170,"events_lines":2,"console_offset":254,"console_lines":1}
+{"time":"2026-08-12T21:57:55.6967349Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:10.573041844Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":152,"history_lines":1,"events_offset":172,"events_lines":2,"console_offset":254,"console_lines":1}
+{"time":"2026-08-12T21:58:10.728134421Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:25.573293709Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":153,"history_lines":2,"events_offset":174,"events_lines":2,"console_offset":254,"console_lines":1}
+{"time":"2026-08-12T21:58:25.70203417Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:40.572910895Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":155,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":257,"console_lines":13}
+{"time":"2026-08-12T21:58:40.702594215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:58:55.573258933Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":156,"history_lines":3,"events_offset":178,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-08-12T21:58:55.69998018Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:59:10.572808812Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":159,"history_lines":1,"events_offset":180,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-08-12T21:59:10.716244539Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T21:59:25.572810198Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":160,"history_lines":2,"events_offset":182,"events_lines":2,"console_offset":266,"console_lines":1}
+{"time":"2026-08-12T21:59:25.68063981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_213624-1sryr5l2/run-1sryr5l2.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50a415efa35953c6c0710234234aeeac59334fd3c45644ec5437de3756a4bf47
-size 983040
+oid sha256:b3d3c3673a410a3a3ec714e9c8efafdd9ea8d765d579a231f9a6bf7f9d401ff6
+size 1277952