diff --git a/.gitattributes b/.gitattributes
index a26ffd2e0f07c02e96881434e954a1316cc9125b..673d802b5adb96481923bab8ab32fd00e610e300 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -111,3 +111,4 @@ zain/Activation/wandb/run-20260819_163845-74tq2syl/run-74tq2syl.wandb filter=lfs
 zain/Activation/wandb/run-20260819_164553-44x03ghu/run-44x03ghu.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260819_170854-zhg4u13t/run-zhg4u13t.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260819_171215-2nil4rqn/run-2nil4rqn.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260819_171541-bhq23mh5/run-bhq23mh5.wandb filter=lfs diff=lfs merge=lfs -text
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/scheduler.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/scheduler.pt
index 0105a5059b2b66b8f19f03ab1912bf3a2be45395..9a65bc49509b0c862cfbc0fbf03e542b72288ba5 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/scheduler.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/scheduler.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f768f3dc231ac5aded23c6f019b283714352126180577c2c811e2d205bc7704f
+oid sha256:af1ea62c89448929159c1b05ebc75e926e6be85e7ce3551310793c351ba3ade1
 size 1064
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/trainer_state.json b/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/trainer_state.json
index fdde6c0f4a29eded0a46aa735c1fa259fd2a607c..4ce78af96432a1f8b93903264060fd05b92608e3 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/trainer_state.json
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/trainer_state.json
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.013481631277384564,
+  "epoch": 0.03370407819346141,
   "eval_steps": 100,
   "global_step": 400,
   "is_hyper_param_search": false,
@@ -10,180 +10,180 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0006740815638692282,
-      "grad_norm": 1.796875,
-      "learning_rate": 1.14e-05,
-      "loss": 8.322640228271485,
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.5e-05,
+      "loss": 8.200721740722656,
       "step": 20
     },
     {
-      "epoch": 0.0013481631277384564,
-      "grad_norm": 1.75,
-      "learning_rate": 2.34e-05,
-      "loss": 8.267462158203125,
+      "epoch": 0.003370407819346141,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.764720153808594,
       "step": 40
     },
     {
-      "epoch": 0.0020222446916076846,
-      "grad_norm": 1.3046875,
-      "learning_rate": 3.539999999999999e-05,
-      "loss": 8.106219482421874,
+      "epoch": 0.005055611729019211,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.160160064697266,
       "step": 60
     },
     {
-      "epoch": 0.002696326255476913,
-      "grad_norm": 1.28125,
-      "learning_rate": 4.7399999999999993e-05,
-      "loss": 7.948130798339844,
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.2109375,
+      "learning_rate": 0.000395,
+      "loss": 6.480591583251953,
       "step": 80
     },
     {
-      "epoch": 0.003370407819346141,
-      "grad_norm": 1.25,
-      "learning_rate": 5.94e-05,
-      "loss": 7.777301788330078,
+      "epoch": 0.008426019548365353,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000495,
+      "loss": 5.919136428833008,
       "step": 100
     },
     {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.679966926574707,
-      "eval_runtime": 8.4901,
-      "eval_samples_per_second": 1122.134,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.008426019548365353,
+      "eval_loss": 5.634258270263672,
+      "eval_runtime": 7.97,
+      "eval_samples_per_second": 1195.356,
+      "eval_steps_per_second": 0.878,
       "step": 100
     },
     {
-      "epoch": 0.004044489383215369,
-      "grad_norm": 1.2578125,
-      "learning_rate": 7.139999999999999e-05,
-      "loss": 7.585383605957031,
+      "epoch": 0.010111223458038422,
+      "grad_norm": 1.171875,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 5.412916946411133,
       "step": 120
     },
     {
-      "epoch": 0.0047185709470845974,
-      "grad_norm": 1.25,
-      "learning_rate": 8.34e-05,
-      "loss": 7.3637535095214846,
+      "epoch": 0.011796427367711493,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.000695,
+      "loss": 5.0327880859375,
       "step": 140
     },
     {
-      "epoch": 0.005392652510953826,
-      "grad_norm": 1.25,
-      "learning_rate": 9.539999999999999e-05,
-      "loss": 7.1375579833984375,
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.000795,
+      "loss": 4.69476089477539,
       "step": 160
     },
     {
-      "epoch": 0.006066734074823054,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00010739999999999998,
-      "loss": 6.910031127929687,
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 4.4246673583984375,
       "step": 180
     },
     {
-      "epoch": 0.006740815638692282,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0001194,
-      "loss": 6.671029663085937,
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.000995,
+      "loss": 4.204695129394532,
       "step": 200
     },
     {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 6.548758506774902,
-      "eval_runtime": 8.6601,
-      "eval_samples_per_second": 1100.105,
-      "eval_steps_per_second": 0.808,
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.133424282073975,
+      "eval_runtime": 7.9329,
+      "eval_samples_per_second": 1200.942,
+      "eval_steps_per_second": 0.882,
       "step": 200
     },
     {
-      "epoch": 0.00741489720256151,
-      "grad_norm": 1.578125,
-      "learning_rate": 0.0001314,
-      "loss": 6.457804107666016,
+      "epoch": 0.018537243006403775,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 4.048733520507812,
       "step": 220
     },
     {
-      "epoch": 0.008088978766430738,
-      "grad_norm": 1.4921875,
-      "learning_rate": 0.0001434,
-      "loss": 6.254596328735351,
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 3.9190834045410154,
       "step": 240
     },
     {
-      "epoch": 0.008763060330299966,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00015539999999999998,
-      "loss": 6.047513961791992,
+      "epoch": 0.021907650825749917,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 3.7833847045898437,
       "step": 260
     },
     {
-      "epoch": 0.009437141894169195,
-      "grad_norm": 1.8828125,
-      "learning_rate": 0.0001674,
-      "loss": 5.870236587524414,
+      "epoch": 0.023592854735422986,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.700960159301758,
       "step": 280
     },
     {
-      "epoch": 0.010111223458038422,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.00017939999999999997,
-      "loss": 5.72708740234375,
+      "epoch": 0.025278058645096056,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.001,
+      "loss": 3.6359439849853517,
       "step": 300
     },
     {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 5.652859687805176,
-      "eval_runtime": 8.4927,
-      "eval_samples_per_second": 1121.794,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.025278058645096056,
+      "eval_loss": 3.5975606441497803,
+      "eval_runtime": 7.973,
+      "eval_samples_per_second": 1194.91,
+      "eval_steps_per_second": 0.878,
       "step": 300
     },
     {
-      "epoch": 0.010785305021907651,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0001914,
-      "loss": 5.585579681396484,
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.5393722534179686,
       "step": 320
     },
     {
-      "epoch": 0.011459386585776879,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.00020339999999999998,
-      "loss": 5.470914077758789,
+      "epoch": 0.028648466464442197,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 3.492219924926758,
       "step": 340
     },
     {
-      "epoch": 0.012133468149646108,
-      "grad_norm": 1.4140625,
-      "learning_rate": 0.00021539999999999998,
-      "loss": 5.318555068969727,
+      "epoch": 0.030333670374115267,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.4430694580078125,
       "step": 360
     },
     {
-      "epoch": 0.012807549713515335,
-      "grad_norm": 1.6015625,
-      "learning_rate": 0.00022739999999999997,
-      "loss": 5.185982894897461,
+      "epoch": 0.032018874283788336,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.3989883422851563,
       "step": 380
     },
     {
-      "epoch": 0.013481631277384564,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.0002394,
-      "loss": 5.065654754638672,
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.341664123535156,
       "step": 400
     },
     {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.008047103881836,
-      "eval_runtime": 8.5503,
-      "eval_samples_per_second": 1114.232,
-      "eval_steps_per_second": 0.819,
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.3295202255249023,
+      "eval_runtime": 8.1687,
+      "eval_samples_per_second": 1166.287,
+      "eval_steps_per_second": 0.857,
       "step": 400
     }
   ],
   "logging_steps": 20,
-  "max_steps": 2500,
+  "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -199,8 +199,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 58077688627200.0,
-  "train_batch_size": 32,
+  "total_flos": 145194221568000.0,
+  "train_batch_size": 80,
   "trial_name": null,
   "trial_params": null
 }
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-400/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/model.safetensors b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/model.safetensors
index 73cab7086892e5bcb2b1ef67b5575f68ffff27c4..1299bd4eb86534aaf4493adbf803cf65ccd05844 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/model.safetensors
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5872fbf4c61dbb81c15291d837bafe638d1bfbbcf6e5867744b7390e99c1ad18
+oid sha256:5712d6f569182e98a2720f28f3ef489d602a3c6473ce0b6e3c4f1f7907e3c8cf
 size 4010544
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/optimizer.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/optimizer.pt
index 4867d3c159aa713e38d9c17a93b1fd79706e2628..95f6a397b8a886c928f421ce765d13d3906ca4eb 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/optimizer.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/optimizer.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45bc91dd0e3cd9af5c4d382e7c9e83a79b2fee89488ede8a44ab5697a568f0d1
+oid sha256:63b2d05dc2871dc45cdfc3ecc136218a493535e4bde6da2a670418746fe34818
 size 8068282
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/scheduler.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/scheduler.pt
index b633db3eb9640b0404e87d311e5d856e3ba08760..99b333f888f250a38fb7c0de8e8f03ceb66d7b1d 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/scheduler.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/scheduler.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7aade0a5cf98168c48557061a263a0ce94127833f42effbb1795ad03b5bd88a5
+oid sha256:b11eb900cb7af9ecb8797771dfa83159f999b3d7b4f3c5b2ff6e19c51da8de4a
 size 1064
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/trainer_state.json b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/trainer_state.json
index ac7c71ffb38b4db7f3e869eaf8cdcb8aa18f4c5a..ba52b461663fb7b911af778c478fb4e2ea5c5714 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/trainer_state.json
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/trainer_state.json
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.016852039096730706,
+  "epoch": 0.04213009774182676,
   "eval_steps": 100,
   "global_step": 500,
   "is_hyper_param_search": false,
@@ -10,223 +10,223 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0006740815638692282,
-      "grad_norm": 1.796875,
-      "learning_rate": 1.14e-05,
-      "loss": 8.322640228271485,
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.5e-05,
+      "loss": 8.200721740722656,
       "step": 20
     },
     {
-      "epoch": 0.0013481631277384564,
-      "grad_norm": 1.75,
-      "learning_rate": 2.34e-05,
-      "loss": 8.267462158203125,
+      "epoch": 0.003370407819346141,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.764720153808594,
       "step": 40
     },
     {
-      "epoch": 0.0020222446916076846,
-      "grad_norm": 1.3046875,
-      "learning_rate": 3.539999999999999e-05,
-      "loss": 8.106219482421874,
+      "epoch": 0.005055611729019211,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.160160064697266,
       "step": 60
     },
     {
-      "epoch": 0.002696326255476913,
-      "grad_norm": 1.28125,
-      "learning_rate": 4.7399999999999993e-05,
-      "loss": 7.948130798339844,
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.2109375,
+      "learning_rate": 0.000395,
+      "loss": 6.480591583251953,
       "step": 80
     },
     {
-      "epoch": 0.003370407819346141,
-      "grad_norm": 1.25,
-      "learning_rate": 5.94e-05,
-      "loss": 7.777301788330078,
+      "epoch": 0.008426019548365353,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000495,
+      "loss": 5.919136428833008,
       "step": 100
     },
     {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.679966926574707,
-      "eval_runtime": 8.4901,
-      "eval_samples_per_second": 1122.134,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.008426019548365353,
+      "eval_loss": 5.634258270263672,
+      "eval_runtime": 7.97,
+      "eval_samples_per_second": 1195.356,
+      "eval_steps_per_second": 0.878,
       "step": 100
     },
     {
-      "epoch": 0.004044489383215369,
-      "grad_norm": 1.2578125,
-      "learning_rate": 7.139999999999999e-05,
-      "loss": 7.585383605957031,
+      "epoch": 0.010111223458038422,
+      "grad_norm": 1.171875,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 5.412916946411133,
       "step": 120
     },
     {
-      "epoch": 0.0047185709470845974,
-      "grad_norm": 1.25,
-      "learning_rate": 8.34e-05,
-      "loss": 7.3637535095214846,
+      "epoch": 0.011796427367711493,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.000695,
+      "loss": 5.0327880859375,
       "step": 140
     },
     {
-      "epoch": 0.005392652510953826,
-      "grad_norm": 1.25,
-      "learning_rate": 9.539999999999999e-05,
-      "loss": 7.1375579833984375,
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.000795,
+      "loss": 4.69476089477539,
       "step": 160
     },
     {
-      "epoch": 0.006066734074823054,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00010739999999999998,
-      "loss": 6.910031127929687,
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 4.4246673583984375,
       "step": 180
     },
     {
-      "epoch": 0.006740815638692282,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0001194,
-      "loss": 6.671029663085937,
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.000995,
+      "loss": 4.204695129394532,
       "step": 200
     },
     {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 6.548758506774902,
-      "eval_runtime": 8.6601,
-      "eval_samples_per_second": 1100.105,
-      "eval_steps_per_second": 0.808,
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.133424282073975,
+      "eval_runtime": 7.9329,
+      "eval_samples_per_second": 1200.942,
+      "eval_steps_per_second": 0.882,
       "step": 200
     },
     {
-      "epoch": 0.00741489720256151,
-      "grad_norm": 1.578125,
-      "learning_rate": 0.0001314,
-      "loss": 6.457804107666016,
+      "epoch": 0.018537243006403775,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 4.048733520507812,
       "step": 220
     },
     {
-      "epoch": 0.008088978766430738,
-      "grad_norm": 1.4921875,
-      "learning_rate": 0.0001434,
-      "loss": 6.254596328735351,
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 3.9190834045410154,
       "step": 240
     },
     {
-      "epoch": 0.008763060330299966,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00015539999999999998,
-      "loss": 6.047513961791992,
+      "epoch": 0.021907650825749917,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 3.7833847045898437,
       "step": 260
     },
     {
-      "epoch": 0.009437141894169195,
-      "grad_norm": 1.8828125,
-      "learning_rate": 0.0001674,
-      "loss": 5.870236587524414,
+      "epoch": 0.023592854735422986,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.700960159301758,
       "step": 280
     },
     {
-      "epoch": 0.010111223458038422,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.00017939999999999997,
-      "loss": 5.72708740234375,
+      "epoch": 0.025278058645096056,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.001,
+      "loss": 3.6359439849853517,
       "step": 300
     },
     {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 5.652859687805176,
-      "eval_runtime": 8.4927,
-      "eval_samples_per_second": 1121.794,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.025278058645096056,
+      "eval_loss": 3.5975606441497803,
+      "eval_runtime": 7.973,
+      "eval_samples_per_second": 1194.91,
+      "eval_steps_per_second": 0.878,
       "step": 300
     },
     {
-      "epoch": 0.010785305021907651,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0001914,
-      "loss": 5.585579681396484,
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.5393722534179686,
       "step": 320
     },
     {
-      "epoch": 0.011459386585776879,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.00020339999999999998,
-      "loss": 5.470914077758789,
+      "epoch": 0.028648466464442197,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 3.492219924926758,
       "step": 340
     },
     {
-      "epoch": 0.012133468149646108,
-      "grad_norm": 1.4140625,
-      "learning_rate": 0.00021539999999999998,
-      "loss": 5.318555068969727,
+      "epoch": 0.030333670374115267,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.4430694580078125,
       "step": 360
     },
     {
-      "epoch": 0.012807549713515335,
-      "grad_norm": 1.6015625,
-      "learning_rate": 0.00022739999999999997,
-      "loss": 5.185982894897461,
+      "epoch": 0.032018874283788336,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.3989883422851563,
       "step": 380
     },
     {
-      "epoch": 0.013481631277384564,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.0002394,
-      "loss": 5.065654754638672,
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.341664123535156,
       "step": 400
     },
     {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.008047103881836,
-      "eval_runtime": 8.5503,
-      "eval_samples_per_second": 1114.232,
-      "eval_steps_per_second": 0.819,
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.3295202255249023,
+      "eval_runtime": 8.1687,
+      "eval_samples_per_second": 1166.287,
+      "eval_steps_per_second": 0.857,
       "step": 400
     },
     {
-      "epoch": 0.014155712841253791,
-      "grad_norm": 2.140625,
-      "learning_rate": 0.0002514,
-      "loss": 4.935818481445312,
+      "epoch": 0.03538928210313448,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 3.3047794342041015,
       "step": 420
     },
     {
-      "epoch": 0.01482979440512302,
-      "grad_norm": 1.1875,
-      "learning_rate": 0.00026339999999999995,
-      "loss": 4.853317642211914,
+      "epoch": 0.03707448601280755,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 3.258700942993164,
       "step": 440
     },
     {
-      "epoch": 0.015503875968992248,
-      "grad_norm": 1.5390625,
-      "learning_rate": 0.00027539999999999997,
-      "loss": 4.7492321014404295,
+      "epoch": 0.03875968992248062,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 3.229313278198242,
       "step": 460
     },
     {
-      "epoch": 0.016177957532861477,
-      "grad_norm": 1.921875,
-      "learning_rate": 0.00028739999999999994,
-      "loss": 4.629489898681641,
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.202067565917969,
       "step": 480
     },
     {
-      "epoch": 0.016852039096730706,
-      "grad_norm": 1.7890625,
-      "learning_rate": 0.00029939999999999996,
-      "loss": 4.565845108032226,
+      "epoch": 0.04213009774182676,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 3.163772201538086,
       "step": 500
     },
     {
-      "epoch": 0.016852039096730706,
-      "eval_loss": 4.530772686004639,
-      "eval_runtime": 8.4284,
-      "eval_samples_per_second": 1130.345,
-      "eval_steps_per_second": 0.831,
+      "epoch": 0.04213009774182676,
+      "eval_loss": 3.157355308532715,
+      "eval_runtime": 7.9503,
+      "eval_samples_per_second": 1198.315,
+      "eval_steps_per_second": 0.88,
       "step": 500
     }
   ],
   "logging_steps": 20,
-  "max_steps": 2500,
+  "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -242,8 +242,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 72597110784000.0,
-  "train_batch_size": 32,
+  "total_flos": 181492776960000.0,
+  "train_batch_size": 80,
   "trial_name": null,
   "trial_params": null
 }
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-500/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/model.safetensors b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/model.safetensors
index f0fa3a664e53bc2a65668866f95df4401eb00fa8..93390206447747eadf363037c291121a96ce3950 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/model.safetensors
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4c2f0e7e1bd95005c2e46113faf5cd14e9b94fc90f937ea9d51aeb93a33a04e
+oid sha256:fdd583528bce16f5f52bc31609f36b44fc4dc0de87540acf616f1442cb07fde9
 size 4010544
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/optimizer.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/optimizer.pt
index 112f4ba540bc8fa4447fee7877f0410d9cd5b58f..7c16a309bda59b191b03ab2f9bbabd39828e4ba9 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/optimizer.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/optimizer.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:946a1da1fa4a395f840276f2b24058ef226b048d8eab3397ee86488c7d82670a
+oid sha256:8fe1053b1cf05e779ceab88f2312865667a8cb4999b4439b1a72908989024233
 size 8068282
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/scheduler.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/scheduler.pt
index 8329d812eb830c2a339cb275b55b5ab2ad3ce019..d9bfb776e6d88a8d083d935888c60c14c3e54429 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/scheduler.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/scheduler.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:295d5fdc0e0bbae04d3db91b9c6051a7cccac49c004495947e95dcd6c59adda3
+oid sha256:31ff2c480a5c30c49d6c8b2f5ae72d7dc39d2d3f7060f88a23705954d049a506
 size 1064
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/trainer_state.json b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/trainer_state.json
index 80ff77b43332ed698124c3a03d7edfffd565c5e8..a3eb6d991080146980ddf3eb19bd07a905b2288b 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/trainer_state.json
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/trainer_state.json
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.020222446916076844,
+  "epoch": 0.05055611729019211,
   "eval_steps": 100,
   "global_step": 600,
   "is_hyper_param_search": false,
@@ -10,266 +10,266 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0006740815638692282,
-      "grad_norm": 1.796875,
-      "learning_rate": 1.14e-05,
-      "loss": 8.322640228271485,
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.5e-05,
+      "loss": 8.200721740722656,
       "step": 20
     },
     {
-      "epoch": 0.0013481631277384564,
-      "grad_norm": 1.75,
-      "learning_rate": 2.34e-05,
-      "loss": 8.267462158203125,
+      "epoch": 0.003370407819346141,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.764720153808594,
       "step": 40
     },
     {
-      "epoch": 0.0020222446916076846,
-      "grad_norm": 1.3046875,
-      "learning_rate": 3.539999999999999e-05,
-      "loss": 8.106219482421874,
+      "epoch": 0.005055611729019211,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.160160064697266,
       "step": 60
     },
     {
-      "epoch": 0.002696326255476913,
-      "grad_norm": 1.28125,
-      "learning_rate": 4.7399999999999993e-05,
-      "loss": 7.948130798339844,
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.2109375,
+      "learning_rate": 0.000395,
+      "loss": 6.480591583251953,
       "step": 80
     },
     {
-      "epoch": 0.003370407819346141,
-      "grad_norm": 1.25,
-      "learning_rate": 5.94e-05,
-      "loss": 7.777301788330078,
+      "epoch": 0.008426019548365353,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000495,
+      "loss": 5.919136428833008,
       "step": 100
     },
     {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.679966926574707,
-      "eval_runtime": 8.4901,
-      "eval_samples_per_second": 1122.134,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.008426019548365353,
+      "eval_loss": 5.634258270263672,
+      "eval_runtime": 7.97,
+      "eval_samples_per_second": 1195.356,
+      "eval_steps_per_second": 0.878,
       "step": 100
     },
     {
-      "epoch": 0.004044489383215369,
-      "grad_norm": 1.2578125,
-      "learning_rate": 7.139999999999999e-05,
-      "loss": 7.585383605957031,
+      "epoch": 0.010111223458038422,
+      "grad_norm": 1.171875,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 5.412916946411133,
       "step": 120
     },
     {
-      "epoch": 0.0047185709470845974,
-      "grad_norm": 1.25,
-      "learning_rate": 8.34e-05,
-      "loss": 7.3637535095214846,
+      "epoch": 0.011796427367711493,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.000695,
+      "loss": 5.0327880859375,
       "step": 140
     },
     {
-      "epoch": 0.005392652510953826,
-      "grad_norm": 1.25,
-      "learning_rate": 9.539999999999999e-05,
-      "loss": 7.1375579833984375,
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.000795,
+      "loss": 4.69476089477539,
       "step": 160
     },
     {
-      "epoch": 0.006066734074823054,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00010739999999999998,
-      "loss": 6.910031127929687,
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 4.4246673583984375,
       "step": 180
     },
     {
-      "epoch": 0.006740815638692282,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0001194,
-      "loss": 6.671029663085937,
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.000995,
+      "loss": 4.204695129394532,
       "step": 200
     },
     {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 6.548758506774902,
-      "eval_runtime": 8.6601,
-      "eval_samples_per_second": 1100.105,
-      "eval_steps_per_second": 0.808,
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.133424282073975,
+      "eval_runtime": 7.9329,
+      "eval_samples_per_second": 1200.942,
+      "eval_steps_per_second": 0.882,
       "step": 200
     },
     {
-      "epoch": 0.00741489720256151,
-      "grad_norm": 1.578125,
-      "learning_rate": 0.0001314,
-      "loss": 6.457804107666016,
+      "epoch": 0.018537243006403775,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 4.048733520507812,
       "step": 220
     },
     {
-      "epoch": 0.008088978766430738,
-      "grad_norm": 1.4921875,
-      "learning_rate": 0.0001434,
-      "loss": 6.254596328735351,
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 3.9190834045410154,
       "step": 240
     },
     {
-      "epoch": 0.008763060330299966,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00015539999999999998,
-      "loss": 6.047513961791992,
+      "epoch": 0.021907650825749917,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 3.7833847045898437,
       "step": 260
     },
     {
-      "epoch": 0.009437141894169195,
-      "grad_norm": 1.8828125,
-      "learning_rate": 0.0001674,
-      "loss": 5.870236587524414,
+      "epoch": 0.023592854735422986,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.700960159301758,
       "step": 280
     },
     {
-      "epoch": 0.010111223458038422,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.00017939999999999997,
-      "loss": 5.72708740234375,
+      "epoch": 0.025278058645096056,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.001,
+      "loss": 3.6359439849853517,
       "step": 300
     },
     {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 5.652859687805176,
-      "eval_runtime": 8.4927,
-      "eval_samples_per_second": 1121.794,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.025278058645096056,
+      "eval_loss": 3.5975606441497803,
+      "eval_runtime": 7.973,
+      "eval_samples_per_second": 1194.91,
+      "eval_steps_per_second": 0.878,
       "step": 300
     },
     {
-      "epoch": 0.010785305021907651,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0001914,
-      "loss": 5.585579681396484,
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.5393722534179686,
       "step": 320
     },
     {
-      "epoch": 0.011459386585776879,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.00020339999999999998,
-      "loss": 5.470914077758789,
+      "epoch": 0.028648466464442197,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 3.492219924926758,
       "step": 340
     },
     {
-      "epoch": 0.012133468149646108,
-      "grad_norm": 1.4140625,
-      "learning_rate": 0.00021539999999999998,
-      "loss": 5.318555068969727,
+      "epoch": 0.030333670374115267,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.4430694580078125,
       "step": 360
     },
     {
-      "epoch": 0.012807549713515335,
-      "grad_norm": 1.6015625,
-      "learning_rate": 0.00022739999999999997,
-      "loss": 5.185982894897461,
+      "epoch": 0.032018874283788336,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.3989883422851563,
       "step": 380
     },
     {
-      "epoch": 0.013481631277384564,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.0002394,
-      "loss": 5.065654754638672,
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.341664123535156,
       "step": 400
     },
     {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.008047103881836,
-      "eval_runtime": 8.5503,
-      "eval_samples_per_second": 1114.232,
-      "eval_steps_per_second": 0.819,
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.3295202255249023,
+      "eval_runtime": 8.1687,
+      "eval_samples_per_second": 1166.287,
+      "eval_steps_per_second": 0.857,
       "step": 400
     },
     {
-      "epoch": 0.014155712841253791,
-      "grad_norm": 2.140625,
-      "learning_rate": 0.0002514,
-      "loss": 4.935818481445312,
+      "epoch": 0.03538928210313448,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 3.3047794342041015,
       "step": 420
     },
     {
-      "epoch": 0.01482979440512302,
-      "grad_norm": 1.1875,
-      "learning_rate": 0.00026339999999999995,
-      "loss": 4.853317642211914,
+      "epoch": 0.03707448601280755,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 3.258700942993164,
       "step": 440
     },
     {
-      "epoch": 0.015503875968992248,
-      "grad_norm": 1.5390625,
-      "learning_rate": 0.00027539999999999997,
-      "loss": 4.7492321014404295,
+      "epoch": 0.03875968992248062,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 3.229313278198242,
       "step": 460
     },
     {
-      "epoch": 0.016177957532861477,
-      "grad_norm": 1.921875,
-      "learning_rate": 0.00028739999999999994,
-      "loss": 4.629489898681641,
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.202067565917969,
       "step": 480
     },
     {
-      "epoch": 0.016852039096730706,
-      "grad_norm": 1.7890625,
-      "learning_rate": 0.00029939999999999996,
-      "loss": 4.565845108032226,
+      "epoch": 0.04213009774182676,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 3.163772201538086,
       "step": 500
     },
     {
-      "epoch": 0.016852039096730706,
-      "eval_loss": 4.530772686004639,
-      "eval_runtime": 8.4284,
-      "eval_samples_per_second": 1130.345,
-      "eval_steps_per_second": 0.831,
+      "epoch": 0.04213009774182676,
+      "eval_loss": 3.157355308532715,
+      "eval_runtime": 7.9503,
+      "eval_samples_per_second": 1198.315,
+      "eval_steps_per_second": 0.88,
       "step": 500
     },
     {
-      "epoch": 0.01752612066059993,
-      "grad_norm": 1.140625,
-      "learning_rate": 0.0003,
-      "loss": 4.50256233215332,
+      "epoch": 0.043815301651499834,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 3.1448848724365233,
       "step": 520
     },
     {
-      "epoch": 0.01820020222446916,
-      "grad_norm": 1.7265625,
-      "learning_rate": 0.0003,
-      "loss": 4.419484710693359,
+      "epoch": 0.0455005055611729,
+      "grad_norm": 0.828125,
+      "learning_rate": 0.001,
+      "loss": 3.103527069091797,
       "step": 540
     },
     {
-      "epoch": 0.01887428378833839,
-      "grad_norm": 1.3125,
-      "learning_rate": 0.0003,
-      "loss": 4.363323593139649,
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.001,
+      "loss": 3.08404541015625,
       "step": 560
     },
     {
-      "epoch": 0.01954836535220762,
-      "grad_norm": 1.5,
-      "learning_rate": 0.0003,
-      "loss": 4.3072765350341795,
+      "epoch": 0.04887091338051904,
+      "grad_norm": 0.76953125,
+      "learning_rate": 0.001,
+      "loss": 3.0501741409301757,
       "step": 580
     },
     {
-      "epoch": 0.020222446916076844,
-      "grad_norm": 1.9765625,
-      "learning_rate": 0.0003,
-      "loss": 4.263522338867188,
+      "epoch": 0.05055611729019211,
+      "grad_norm": 0.96875,
+      "learning_rate": 0.001,
+      "loss": 3.0376760482788088,
       "step": 600
     },
     {
-      "epoch": 0.020222446916076844,
-      "eval_loss": 4.239859580993652,
-      "eval_runtime": 8.5836,
-      "eval_samples_per_second": 1109.908,
-      "eval_steps_per_second": 0.816,
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.0310890674591064,
+      "eval_runtime": 8.1195,
+      "eval_samples_per_second": 1173.346,
+      "eval_steps_per_second": 0.862,
       "step": 600
     }
   ],
   "logging_steps": 20,
-  "max_steps": 2500,
+  "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -285,8 +285,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 87116532940800.0,
-  "train_batch_size": 32,
+  "total_flos": 217791332352000.0,
+  "train_batch_size": 80,
   "trial_name": null,
   "trial_params": null
 }
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-600/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/model.safetensors b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/model.safetensors
index d52ad380cbcc3f05cf95ddfa9ee6b98ea65a990a..e685a48e47e11372131ad91af89ba2bbdfefa21e 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/model.safetensors
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1dce8189770ce838ce3e65dd567433fe02803f116d52a94c356237378a46e5c1
+oid sha256:97deb4d2fe5491f671435ead6997073978dc5974641a5f915ea901926274c1a1
 size 4010544
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/optimizer.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/optimizer.pt
index 7c7ce6e7b3ddc694f95deaaa7ba6ed8eefa9cfb8..b3d7fcb32431e03e866e9e9fb82e33822a3b877a 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/optimizer.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/optimizer.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58e5fb749b87b488b4c7fc432602eba6ba868802571d6b14fdf6a60ec4240857
+oid sha256:1f5e62abb7b4309d85d8e7e3f37e3449916b0bac155e62f602e3bbd0aa0a7951
 size 8068282
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/scheduler.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/scheduler.pt
index e7b9e12be6722ec6066dd0695e2e0323ffceb44e..4f74867bf74bdb2ec29036dae378fe9753db729c 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/scheduler.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/scheduler.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6c38b4e6128ebabd81a22bafe942090015968e75f90e5e9ef25cd5ebdef8357
+oid sha256:dd7c397523408b6e567c3990f49ad726aa8ff2c476b37da2c996606b1b8ddb75
 size 1064
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/trainer_state.json b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/trainer_state.json
index 3e01ddde2bf38d261df49a609a384413608aaba9..2d9011b70a41db72ae838ae16f18a0e06b01f65d 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/trainer_state.json
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/trainer_state.json
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.023592854735422986,
+  "epoch": 0.058982136838557464,
   "eval_steps": 100,
   "global_step": 700,
   "is_hyper_param_search": false,
@@ -10,309 +10,309 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0006740815638692282,
-      "grad_norm": 1.796875,
-      "learning_rate": 1.14e-05,
-      "loss": 8.322640228271485,
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.5e-05,
+      "loss": 8.200721740722656,
       "step": 20
     },
     {
-      "epoch": 0.0013481631277384564,
-      "grad_norm": 1.75,
-      "learning_rate": 2.34e-05,
-      "loss": 8.267462158203125,
+      "epoch": 0.003370407819346141,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.764720153808594,
       "step": 40
     },
     {
-      "epoch": 0.0020222446916076846,
-      "grad_norm": 1.3046875,
-      "learning_rate": 3.539999999999999e-05,
-      "loss": 8.106219482421874,
+      "epoch": 0.005055611729019211,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.160160064697266,
       "step": 60
     },
     {
-      "epoch": 0.002696326255476913,
-      "grad_norm": 1.28125,
-      "learning_rate": 4.7399999999999993e-05,
-      "loss": 7.948130798339844,
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.2109375,
+      "learning_rate": 0.000395,
+      "loss": 6.480591583251953,
       "step": 80
     },
     {
-      "epoch": 0.003370407819346141,
-      "grad_norm": 1.25,
-      "learning_rate": 5.94e-05,
-      "loss": 7.777301788330078,
+      "epoch": 0.008426019548365353,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000495,
+      "loss": 5.919136428833008,
       "step": 100
     },
     {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.679966926574707,
-      "eval_runtime": 8.4901,
-      "eval_samples_per_second": 1122.134,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.008426019548365353,
+      "eval_loss": 5.634258270263672,
+      "eval_runtime": 7.97,
+      "eval_samples_per_second": 1195.356,
+      "eval_steps_per_second": 0.878,
       "step": 100
     },
     {
-      "epoch": 0.004044489383215369,
-      "grad_norm": 1.2578125,
-      "learning_rate": 7.139999999999999e-05,
-      "loss": 7.585383605957031,
+      "epoch": 0.010111223458038422,
+      "grad_norm": 1.171875,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 5.412916946411133,
       "step": 120
     },
     {
-      "epoch": 0.0047185709470845974,
-      "grad_norm": 1.25,
-      "learning_rate": 8.34e-05,
-      "loss": 7.3637535095214846,
+      "epoch": 0.011796427367711493,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.000695,
+      "loss": 5.0327880859375,
       "step": 140
     },
     {
-      "epoch": 0.005392652510953826,
-      "grad_norm": 1.25,
-      "learning_rate": 9.539999999999999e-05,
-      "loss": 7.1375579833984375,
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.000795,
+      "loss": 4.69476089477539,
       "step": 160
     },
     {
-      "epoch": 0.006066734074823054,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00010739999999999998,
-      "loss": 6.910031127929687,
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 4.4246673583984375,
       "step": 180
     },
     {
-      "epoch": 0.006740815638692282,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0001194,
-      "loss": 6.671029663085937,
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.000995,
+      "loss": 4.204695129394532,
       "step": 200
     },
     {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 6.548758506774902,
-      "eval_runtime": 8.6601,
-      "eval_samples_per_second": 1100.105,
-      "eval_steps_per_second": 0.808,
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.133424282073975,
+      "eval_runtime": 7.9329,
+      "eval_samples_per_second": 1200.942,
+      "eval_steps_per_second": 0.882,
       "step": 200
     },
     {
-      "epoch": 0.00741489720256151,
-      "grad_norm": 1.578125,
-      "learning_rate": 0.0001314,
-      "loss": 6.457804107666016,
+      "epoch": 0.018537243006403775,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 4.048733520507812,
       "step": 220
     },
     {
-      "epoch": 0.008088978766430738,
-      "grad_norm": 1.4921875,
-      "learning_rate": 0.0001434,
-      "loss": 6.254596328735351,
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 3.9190834045410154,
       "step": 240
     },
     {
-      "epoch": 0.008763060330299966,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00015539999999999998,
-      "loss": 6.047513961791992,
+      "epoch": 0.021907650825749917,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 3.7833847045898437,
       "step": 260
     },
     {
-      "epoch": 0.009437141894169195,
-      "grad_norm": 1.8828125,
-      "learning_rate": 0.0001674,
-      "loss": 5.870236587524414,
+      "epoch": 0.023592854735422986,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.700960159301758,
       "step": 280
     },
     {
-      "epoch": 0.010111223458038422,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.00017939999999999997,
-      "loss": 5.72708740234375,
+      "epoch": 0.025278058645096056,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.001,
+      "loss": 3.6359439849853517,
       "step": 300
     },
     {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 5.652859687805176,
-      "eval_runtime": 8.4927,
-      "eval_samples_per_second": 1121.794,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.025278058645096056,
+      "eval_loss": 3.5975606441497803,
+      "eval_runtime": 7.973,
+      "eval_samples_per_second": 1194.91,
+      "eval_steps_per_second": 0.878,
       "step": 300
     },
     {
-      "epoch": 0.010785305021907651,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0001914,
-      "loss": 5.585579681396484,
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.5393722534179686,
       "step": 320
     },
     {
-      "epoch": 0.011459386585776879,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.00020339999999999998,
-      "loss": 5.470914077758789,
+      "epoch": 0.028648466464442197,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 3.492219924926758,
       "step": 340
     },
     {
-      "epoch": 0.012133468149646108,
-      "grad_norm": 1.4140625,
-      "learning_rate": 0.00021539999999999998,
-      "loss": 5.318555068969727,
+      "epoch": 0.030333670374115267,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.4430694580078125,
       "step": 360
     },
     {
-      "epoch": 0.012807549713515335,
-      "grad_norm": 1.6015625,
-      "learning_rate": 0.00022739999999999997,
-      "loss": 5.185982894897461,
+      "epoch": 0.032018874283788336,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.3989883422851563,
       "step": 380
     },
     {
-      "epoch": 0.013481631277384564,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.0002394,
-      "loss": 5.065654754638672,
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.341664123535156,
       "step": 400
     },
     {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.008047103881836,
-      "eval_runtime": 8.5503,
-      "eval_samples_per_second": 1114.232,
-      "eval_steps_per_second": 0.819,
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.3295202255249023,
+      "eval_runtime": 8.1687,
+      "eval_samples_per_second": 1166.287,
+      "eval_steps_per_second": 0.857,
       "step": 400
     },
     {
-      "epoch": 0.014155712841253791,
-      "grad_norm": 2.140625,
-      "learning_rate": 0.0002514,
-      "loss": 4.935818481445312,
+      "epoch": 0.03538928210313448,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 3.3047794342041015,
       "step": 420
     },
     {
-      "epoch": 0.01482979440512302,
-      "grad_norm": 1.1875,
-      "learning_rate": 0.00026339999999999995,
-      "loss": 4.853317642211914,
+      "epoch": 0.03707448601280755,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 3.258700942993164,
       "step": 440
     },
     {
-      "epoch": 0.015503875968992248,
-      "grad_norm": 1.5390625,
-      "learning_rate": 0.00027539999999999997,
-      "loss": 4.7492321014404295,
+      "epoch": 0.03875968992248062,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 3.229313278198242,
       "step": 460
     },
     {
-      "epoch": 0.016177957532861477,
-      "grad_norm": 1.921875,
-      "learning_rate": 0.00028739999999999994,
-      "loss": 4.629489898681641,
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.202067565917969,
       "step": 480
     },
     {
-      "epoch": 0.016852039096730706,
-      "grad_norm": 1.7890625,
-      "learning_rate": 0.00029939999999999996,
-      "loss": 4.565845108032226,
+      "epoch": 0.04213009774182676,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 3.163772201538086,
       "step": 500
     },
     {
-      "epoch": 0.016852039096730706,
-      "eval_loss": 4.530772686004639,
-      "eval_runtime": 8.4284,
-      "eval_samples_per_second": 1130.345,
-      "eval_steps_per_second": 0.831,
+      "epoch": 0.04213009774182676,
+      "eval_loss": 3.157355308532715,
+      "eval_runtime": 7.9503,
+      "eval_samples_per_second": 1198.315,
+      "eval_steps_per_second": 0.88,
       "step": 500
     },
     {
-      "epoch": 0.01752612066059993,
-      "grad_norm": 1.140625,
-      "learning_rate": 0.0003,
-      "loss": 4.50256233215332,
+      "epoch": 0.043815301651499834,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 3.1448848724365233,
       "step": 520
     },
     {
-      "epoch": 0.01820020222446916,
-      "grad_norm": 1.7265625,
-      "learning_rate": 0.0003,
-      "loss": 4.419484710693359,
+      "epoch": 0.0455005055611729,
+      "grad_norm": 0.828125,
+      "learning_rate": 0.001,
+      "loss": 3.103527069091797,
       "step": 540
     },
     {
-      "epoch": 0.01887428378833839,
-      "grad_norm": 1.3125,
-      "learning_rate": 0.0003,
-      "loss": 4.363323593139649,
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.001,
+      "loss": 3.08404541015625,
       "step": 560
     },
     {
-      "epoch": 0.01954836535220762,
-      "grad_norm": 1.5,
-      "learning_rate": 0.0003,
-      "loss": 4.3072765350341795,
+      "epoch": 0.04887091338051904,
+      "grad_norm": 0.76953125,
+      "learning_rate": 0.001,
+      "loss": 3.0501741409301757,
       "step": 580
     },
     {
-      "epoch": 0.020222446916076844,
-      "grad_norm": 1.9765625,
-      "learning_rate": 0.0003,
-      "loss": 4.263522338867188,
+      "epoch": 0.05055611729019211,
+      "grad_norm": 0.96875,
+      "learning_rate": 0.001,
+      "loss": 3.0376760482788088,
       "step": 600
     },
     {
-      "epoch": 0.020222446916076844,
-      "eval_loss": 4.239859580993652,
-      "eval_runtime": 8.5836,
-      "eval_samples_per_second": 1109.908,
-      "eval_steps_per_second": 0.816,
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.0310890674591064,
+      "eval_runtime": 8.1195,
+      "eval_samples_per_second": 1173.346,
+      "eval_steps_per_second": 0.862,
       "step": 600
     },
     {
-      "epoch": 0.020896528479946073,
-      "grad_norm": 1.2265625,
-      "learning_rate": 0.0003,
-      "loss": 4.207575988769531,
+      "epoch": 0.05224132119986518,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.0314205169677733,
       "step": 620
     },
     {
-      "epoch": 0.021570610043815303,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0003,
-      "loss": 4.179453659057617,
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.640625,
+      "learning_rate": 0.001,
+      "loss": 3.0014928817749023,
       "step": 640
     },
     {
-      "epoch": 0.022244691607684528,
-      "grad_norm": 1.09375,
-      "learning_rate": 0.0003,
-      "loss": 4.13963508605957,
+      "epoch": 0.055611729019211326,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.001,
+      "loss": 2.9963293075561523,
       "step": 660
     },
     {
-      "epoch": 0.022918773171553757,
-      "grad_norm": 1.328125,
-      "learning_rate": 0.0003,
-      "loss": 4.0913646697998045,
+      "epoch": 0.057296932928884395,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 2.9568761825561523,
       "step": 680
     },
     {
-      "epoch": 0.023592854735422986,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0003,
-      "loss": 4.082810974121093,
+      "epoch": 0.058982136838557464,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.9395275115966797,
       "step": 700
     },
     {
-      "epoch": 0.023592854735422986,
-      "eval_loss": 4.061285495758057,
-      "eval_runtime": 8.3862,
-      "eval_samples_per_second": 1136.028,
-      "eval_steps_per_second": 0.835,
+      "epoch": 0.058982136838557464,
+      "eval_loss": 2.939429759979248,
+      "eval_runtime": 7.975,
+      "eval_samples_per_second": 1194.602,
+      "eval_steps_per_second": 0.878,
       "step": 700
     }
   ],
   "logging_steps": 20,
-  "max_steps": 2500,
+  "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -328,8 +328,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 101635955097600.0,
-  "train_batch_size": 32,
+  "total_flos": 254089887744000.0,
+  "train_batch_size": 80,
   "trial_name": null,
   "trial_params": null
 }
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-700/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/model.safetensors b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/model.safetensors
index fb7b08d3c254ab5187bcc360ff526c06d828664d..3a64a88843d7d781e46390f62332f2a7e6feee99 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/model.safetensors
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75a3bd4fb21fd4ee5917ef4a3effa2e4004eaa865437229e112bf6ed25224797
+oid sha256:9c337edbdecc4bf136ef452013ba3dadf0cb21c44e46ffe85d9ad094448bd625
 size 4010544
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/optimizer.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/optimizer.pt
index fd1c600b75ee398f84c1873e9687215c0a034e9f..4a71a9d025d8d511d6ec0209885d59a048c8d6ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/optimizer.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/optimizer.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32854c5978c8b2f1d33b5d75fb09ef38c174912a2701efac761ca76686bc3446
+oid sha256:99e89d2c66df90ceb6b05584fce25f5cd08caf172652dfb64c617b95863d99bf
 size 8068282
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/scheduler.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/scheduler.pt
index f1b8b80f53c325e1e9a0aa623b0e9e1a16227077..dd97f750f7cfb91f4dce81dc17adb42482a1168e 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/scheduler.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/scheduler.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2224470b783e3f05273b6a4ac609fa6ba76b1099ae64a843a85fea2d2bf1e0dd
+oid sha256:71e4310b54de76d098945960239794c83dba2710505618327c0dcd1468c3497f
 size 1064
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/trainer_state.json b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/trainer_state.json
index ee5a5eaaa6f03fdee737a08e3832311565a5668b..81b5b234c0e279af6277fdf01e474cea8ebd2a53 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/trainer_state.json
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/trainer_state.json
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.026963262554769128,
+  "epoch": 0.06740815638692282,
   "eval_steps": 100,
   "global_step": 800,
   "is_hyper_param_search": false,
@@ -10,352 +10,352 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0006740815638692282,
-      "grad_norm": 1.796875,
-      "learning_rate": 1.14e-05,
-      "loss": 8.322640228271485,
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.5e-05,
+      "loss": 8.200721740722656,
       "step": 20
     },
     {
-      "epoch": 0.0013481631277384564,
-      "grad_norm": 1.75,
-      "learning_rate": 2.34e-05,
-      "loss": 8.267462158203125,
+      "epoch": 0.003370407819346141,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.764720153808594,
       "step": 40
     },
     {
-      "epoch": 0.0020222446916076846,
-      "grad_norm": 1.3046875,
-      "learning_rate": 3.539999999999999e-05,
-      "loss": 8.106219482421874,
+      "epoch": 0.005055611729019211,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.160160064697266,
       "step": 60
     },
     {
-      "epoch": 0.002696326255476913,
-      "grad_norm": 1.28125,
-      "learning_rate": 4.7399999999999993e-05,
-      "loss": 7.948130798339844,
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.2109375,
+      "learning_rate": 0.000395,
+      "loss": 6.480591583251953,
       "step": 80
     },
     {
-      "epoch": 0.003370407819346141,
-      "grad_norm": 1.25,
-      "learning_rate": 5.94e-05,
-      "loss": 7.777301788330078,
+      "epoch": 0.008426019548365353,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000495,
+      "loss": 5.919136428833008,
       "step": 100
     },
     {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.679966926574707,
-      "eval_runtime": 8.4901,
-      "eval_samples_per_second": 1122.134,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.008426019548365353,
+      "eval_loss": 5.634258270263672,
+      "eval_runtime": 7.97,
+      "eval_samples_per_second": 1195.356,
+      "eval_steps_per_second": 0.878,
       "step": 100
     },
     {
-      "epoch": 0.004044489383215369,
-      "grad_norm": 1.2578125,
-      "learning_rate": 7.139999999999999e-05,
-      "loss": 7.585383605957031,
+      "epoch": 0.010111223458038422,
+      "grad_norm": 1.171875,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 5.412916946411133,
       "step": 120
     },
     {
-      "epoch": 0.0047185709470845974,
-      "grad_norm": 1.25,
-      "learning_rate": 8.34e-05,
-      "loss": 7.3637535095214846,
+      "epoch": 0.011796427367711493,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.000695,
+      "loss": 5.0327880859375,
       "step": 140
     },
     {
-      "epoch": 0.005392652510953826,
-      "grad_norm": 1.25,
-      "learning_rate": 9.539999999999999e-05,
-      "loss": 7.1375579833984375,
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.000795,
+      "loss": 4.69476089477539,
       "step": 160
     },
     {
-      "epoch": 0.006066734074823054,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00010739999999999998,
-      "loss": 6.910031127929687,
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 4.4246673583984375,
       "step": 180
     },
     {
-      "epoch": 0.006740815638692282,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0001194,
-      "loss": 6.671029663085937,
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.000995,
+      "loss": 4.204695129394532,
       "step": 200
     },
     {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 6.548758506774902,
-      "eval_runtime": 8.6601,
-      "eval_samples_per_second": 1100.105,
-      "eval_steps_per_second": 0.808,
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.133424282073975,
+      "eval_runtime": 7.9329,
+      "eval_samples_per_second": 1200.942,
+      "eval_steps_per_second": 0.882,
       "step": 200
     },
     {
-      "epoch": 0.00741489720256151,
-      "grad_norm": 1.578125,
-      "learning_rate": 0.0001314,
-      "loss": 6.457804107666016,
+      "epoch": 0.018537243006403775,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 4.048733520507812,
       "step": 220
     },
     {
-      "epoch": 0.008088978766430738,
-      "grad_norm": 1.4921875,
-      "learning_rate": 0.0001434,
-      "loss": 6.254596328735351,
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 3.9190834045410154,
       "step": 240
     },
     {
-      "epoch": 0.008763060330299966,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00015539999999999998,
-      "loss": 6.047513961791992,
+      "epoch": 0.021907650825749917,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 3.7833847045898437,
       "step": 260
     },
     {
-      "epoch": 0.009437141894169195,
-      "grad_norm": 1.8828125,
-      "learning_rate": 0.0001674,
-      "loss": 5.870236587524414,
+      "epoch": 0.023592854735422986,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.700960159301758,
       "step": 280
     },
     {
-      "epoch": 0.010111223458038422,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.00017939999999999997,
-      "loss": 5.72708740234375,
+      "epoch": 0.025278058645096056,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.001,
+      "loss": 3.6359439849853517,
       "step": 300
     },
     {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 5.652859687805176,
-      "eval_runtime": 8.4927,
-      "eval_samples_per_second": 1121.794,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.025278058645096056,
+      "eval_loss": 3.5975606441497803,
+      "eval_runtime": 7.973,
+      "eval_samples_per_second": 1194.91,
+      "eval_steps_per_second": 0.878,
       "step": 300
     },
     {
-      "epoch": 0.010785305021907651,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0001914,
-      "loss": 5.585579681396484,
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.5393722534179686,
       "step": 320
     },
     {
-      "epoch": 0.011459386585776879,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.00020339999999999998,
-      "loss": 5.470914077758789,
+      "epoch": 0.028648466464442197,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 3.492219924926758,
       "step": 340
     },
     {
-      "epoch": 0.012133468149646108,
-      "grad_norm": 1.4140625,
-      "learning_rate": 0.00021539999999999998,
-      "loss": 5.318555068969727,
+      "epoch": 0.030333670374115267,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.4430694580078125,
       "step": 360
     },
     {
-      "epoch": 0.012807549713515335,
-      "grad_norm": 1.6015625,
-      "learning_rate": 0.00022739999999999997,
-      "loss": 5.185982894897461,
+      "epoch": 0.032018874283788336,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.3989883422851563,
       "step": 380
     },
     {
-      "epoch": 0.013481631277384564,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.0002394,
-      "loss": 5.065654754638672,
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.341664123535156,
       "step": 400
     },
     {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.008047103881836,
-      "eval_runtime": 8.5503,
-      "eval_samples_per_second": 1114.232,
-      "eval_steps_per_second": 0.819,
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.3295202255249023,
+      "eval_runtime": 8.1687,
+      "eval_samples_per_second": 1166.287,
+      "eval_steps_per_second": 0.857,
       "step": 400
     },
     {
-      "epoch": 0.014155712841253791,
-      "grad_norm": 2.140625,
-      "learning_rate": 0.0002514,
-      "loss": 4.935818481445312,
+      "epoch": 0.03538928210313448,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 3.3047794342041015,
       "step": 420
     },
     {
-      "epoch": 0.01482979440512302,
-      "grad_norm": 1.1875,
-      "learning_rate": 0.00026339999999999995,
-      "loss": 4.853317642211914,
+      "epoch": 0.03707448601280755,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 3.258700942993164,
       "step": 440
     },
     {
-      "epoch": 0.015503875968992248,
-      "grad_norm": 1.5390625,
-      "learning_rate": 0.00027539999999999997,
-      "loss": 4.7492321014404295,
+      "epoch": 0.03875968992248062,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 3.229313278198242,
       "step": 460
     },
     {
-      "epoch": 0.016177957532861477,
-      "grad_norm": 1.921875,
-      "learning_rate": 0.00028739999999999994,
-      "loss": 4.629489898681641,
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.202067565917969,
       "step": 480
     },
     {
-      "epoch": 0.016852039096730706,
-      "grad_norm": 1.7890625,
-      "learning_rate": 0.00029939999999999996,
-      "loss": 4.565845108032226,
+      "epoch": 0.04213009774182676,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 3.163772201538086,
       "step": 500
     },
     {
-      "epoch": 0.016852039096730706,
-      "eval_loss": 4.530772686004639,
-      "eval_runtime": 8.4284,
-      "eval_samples_per_second": 1130.345,
-      "eval_steps_per_second": 0.831,
+      "epoch": 0.04213009774182676,
+      "eval_loss": 3.157355308532715,
+      "eval_runtime": 7.9503,
+      "eval_samples_per_second": 1198.315,
+      "eval_steps_per_second": 0.88,
       "step": 500
     },
     {
-      "epoch": 0.01752612066059993,
-      "grad_norm": 1.140625,
-      "learning_rate": 0.0003,
-      "loss": 4.50256233215332,
+      "epoch": 0.043815301651499834,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 3.1448848724365233,
       "step": 520
     },
     {
-      "epoch": 0.01820020222446916,
-      "grad_norm": 1.7265625,
-      "learning_rate": 0.0003,
-      "loss": 4.419484710693359,
+      "epoch": 0.0455005055611729,
+      "grad_norm": 0.828125,
+      "learning_rate": 0.001,
+      "loss": 3.103527069091797,
       "step": 540
     },
     {
-      "epoch": 0.01887428378833839,
-      "grad_norm": 1.3125,
-      "learning_rate": 0.0003,
-      "loss": 4.363323593139649,
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.001,
+      "loss": 3.08404541015625,
       "step": 560
     },
     {
-      "epoch": 0.01954836535220762,
-      "grad_norm": 1.5,
-      "learning_rate": 0.0003,
-      "loss": 4.3072765350341795,
+      "epoch": 0.04887091338051904,
+      "grad_norm": 0.76953125,
+      "learning_rate": 0.001,
+      "loss": 3.0501741409301757,
       "step": 580
     },
     {
-      "epoch": 0.020222446916076844,
-      "grad_norm": 1.9765625,
-      "learning_rate": 0.0003,
-      "loss": 4.263522338867188,
+      "epoch": 0.05055611729019211,
+      "grad_norm": 0.96875,
+      "learning_rate": 0.001,
+      "loss": 3.0376760482788088,
       "step": 600
     },
     {
-      "epoch": 0.020222446916076844,
-      "eval_loss": 4.239859580993652,
-      "eval_runtime": 8.5836,
-      "eval_samples_per_second": 1109.908,
-      "eval_steps_per_second": 0.816,
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.0310890674591064,
+      "eval_runtime": 8.1195,
+      "eval_samples_per_second": 1173.346,
+      "eval_steps_per_second": 0.862,
       "step": 600
     },
     {
-      "epoch": 0.020896528479946073,
-      "grad_norm": 1.2265625,
-      "learning_rate": 0.0003,
-      "loss": 4.207575988769531,
+      "epoch": 0.05224132119986518,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.0314205169677733,
       "step": 620
     },
     {
-      "epoch": 0.021570610043815303,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0003,
-      "loss": 4.179453659057617,
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.640625,
+      "learning_rate": 0.001,
+      "loss": 3.0014928817749023,
       "step": 640
     },
     {
-      "epoch": 0.022244691607684528,
-      "grad_norm": 1.09375,
-      "learning_rate": 0.0003,
-      "loss": 4.13963508605957,
+      "epoch": 0.055611729019211326,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.001,
+      "loss": 2.9963293075561523,
       "step": 660
     },
     {
-      "epoch": 0.022918773171553757,
-      "grad_norm": 1.328125,
-      "learning_rate": 0.0003,
-      "loss": 4.0913646697998045,
+      "epoch": 0.057296932928884395,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 2.9568761825561523,
       "step": 680
     },
     {
-      "epoch": 0.023592854735422986,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0003,
-      "loss": 4.082810974121093,
+      "epoch": 0.058982136838557464,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.9395275115966797,
       "step": 700
     },
     {
-      "epoch": 0.023592854735422986,
-      "eval_loss": 4.061285495758057,
-      "eval_runtime": 8.3862,
-      "eval_samples_per_second": 1136.028,
-      "eval_steps_per_second": 0.835,
+      "epoch": 0.058982136838557464,
+      "eval_loss": 2.939429759979248,
+      "eval_runtime": 7.975,
+      "eval_samples_per_second": 1194.602,
+      "eval_steps_per_second": 0.878,
       "step": 700
     },
     {
-      "epoch": 0.024266936299292215,
-      "grad_norm": 1.453125,
-      "learning_rate": 0.0003,
-      "loss": 4.052593612670899,
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 2.922671890258789,
       "step": 720
     },
     {
-      "epoch": 0.02494101786316144,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.0003,
-      "loss": 4.038698196411133,
+      "epoch": 0.06235254465790361,
+      "grad_norm": 0.95703125,
+      "learning_rate": 0.001,
+      "loss": 2.9145633697509767,
       "step": 740
     },
     {
-      "epoch": 0.02561509942703067,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.0003,
-      "loss": 3.99466552734375,
+      "epoch": 0.06403774856757667,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 2.8876668930053713,
       "step": 760
     },
     {
-      "epoch": 0.0262891809908999,
-      "grad_norm": 1.3671875,
-      "learning_rate": 0.0003,
-      "loss": 3.9530941009521485,
+      "epoch": 0.06572295247724974,
+      "grad_norm": 0.69140625,
+      "learning_rate": 0.001,
+      "loss": 2.8692466735839846,
       "step": 780
     },
     {
-      "epoch": 0.026963262554769128,
-      "grad_norm": 1.4375,
-      "learning_rate": 0.0003,
-      "loss": 3.95872802734375,
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 2.8788633346557617,
       "step": 800
     },
     {
-      "epoch": 0.026963262554769128,
-      "eval_loss": 3.9487836360931396,
-      "eval_runtime": 8.7643,
-      "eval_samples_per_second": 1087.026,
-      "eval_steps_per_second": 0.799,
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.8632190227508545,
+      "eval_runtime": 8.2773,
+      "eval_samples_per_second": 1150.973,
+      "eval_steps_per_second": 0.846,
       "step": 800
     }
   ],
   "logging_steps": 20,
-  "max_steps": 2500,
+  "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -371,8 +371,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 116155377254400.0,
-  "train_batch_size": 32,
+  "total_flos": 290388443136000.0,
+  "train_batch_size": 80,
   "trial_name": null,
   "trial_params": null
 }
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-800/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/model.safetensors b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/model.safetensors
index d3f7bd4db9d033603f74d51e1745204cf60c2a7a..968b19137d6b13b2fa8d963ce0d5b37ede77a817 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/model.safetensors
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4137a389ac7c7428d4244efe52cec11b1ef7635b5e3ab39a01d3009f59c58e47
+oid sha256:82e06f7126a42b84896bee7f96709259393d400ad27e92239a7c07ad6c0e0fd4
 size 4010544
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/optimizer.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/optimizer.pt
index 732e03002c664dfa80300692af72c65175d9027b..c853710862d5f33fbad0a2b9517cb8a96af2f213 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/optimizer.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/optimizer.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c462ec1376bb41ba70524306b41158fb5e54895755f46744c0cd2a70fd91f9b
+oid sha256:412b1a58a5c56f4c0d4c86023f862cac2da0249009c2d10b79e1506090879e57
 size 8068282
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/scheduler.pt b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/scheduler.pt
index f8644e228860c805fb609ef30ac9a97bbf737b07..5d1338bdf947c95f24cd90109e2bf061a0218afd 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/scheduler.pt
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/scheduler.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:389b3459cdd290ee62a7be41d029a930e2587531b43c7e5ade84614c6ebc3507
+oid sha256:819e7677327ec819a829273ee6bf375a38913eb519826c73009d0dd3825f5480
 size 1064
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/trainer_state.json b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/trainer_state.json
index 85bf23479c79b47ffd7560f19d3401a890ca1548..1b4979a711390d457358012d183dbe708252fb38 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/trainer_state.json
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/trainer_state.json
@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.030333670374115267,
+  "epoch": 0.07583417593528817,
   "eval_steps": 100,
   "global_step": 900,
   "is_hyper_param_search": false,
@@ -10,395 +10,395 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0006740815638692282,
-      "grad_norm": 1.796875,
-      "learning_rate": 1.14e-05,
-      "loss": 8.322640228271485,
+      "epoch": 0.0016852039096730705,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.5e-05,
+      "loss": 8.200721740722656,
       "step": 20
     },
     {
-      "epoch": 0.0013481631277384564,
-      "grad_norm": 1.75,
-      "learning_rate": 2.34e-05,
-      "loss": 8.267462158203125,
+      "epoch": 0.003370407819346141,
+      "grad_norm": 1.2421875,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 7.764720153808594,
       "step": 40
     },
     {
-      "epoch": 0.0020222446916076846,
-      "grad_norm": 1.3046875,
-      "learning_rate": 3.539999999999999e-05,
-      "loss": 8.106219482421874,
+      "epoch": 0.005055611729019211,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.000295,
+      "loss": 7.160160064697266,
       "step": 60
     },
     {
-      "epoch": 0.002696326255476913,
-      "grad_norm": 1.28125,
-      "learning_rate": 4.7399999999999993e-05,
-      "loss": 7.948130798339844,
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.2109375,
+      "learning_rate": 0.000395,
+      "loss": 6.480591583251953,
       "step": 80
     },
     {
-      "epoch": 0.003370407819346141,
-      "grad_norm": 1.25,
-      "learning_rate": 5.94e-05,
-      "loss": 7.777301788330078,
+      "epoch": 0.008426019548365353,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000495,
+      "loss": 5.919136428833008,
       "step": 100
     },
     {
-      "epoch": 0.003370407819346141,
-      "eval_loss": 7.679966926574707,
-      "eval_runtime": 8.4901,
-      "eval_samples_per_second": 1122.134,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.008426019548365353,
+      "eval_loss": 5.634258270263672,
+      "eval_runtime": 7.97,
+      "eval_samples_per_second": 1195.356,
+      "eval_steps_per_second": 0.878,
       "step": 100
     },
     {
-      "epoch": 0.004044489383215369,
-      "grad_norm": 1.2578125,
-      "learning_rate": 7.139999999999999e-05,
-      "loss": 7.585383605957031,
+      "epoch": 0.010111223458038422,
+      "grad_norm": 1.171875,
+      "learning_rate": 0.0005949999999999999,
+      "loss": 5.412916946411133,
       "step": 120
     },
     {
-      "epoch": 0.0047185709470845974,
-      "grad_norm": 1.25,
-      "learning_rate": 8.34e-05,
-      "loss": 7.3637535095214846,
+      "epoch": 0.011796427367711493,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.000695,
+      "loss": 5.0327880859375,
       "step": 140
     },
     {
-      "epoch": 0.005392652510953826,
-      "grad_norm": 1.25,
-      "learning_rate": 9.539999999999999e-05,
-      "loss": 7.1375579833984375,
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.000795,
+      "loss": 4.69476089477539,
       "step": 160
     },
     {
-      "epoch": 0.006066734074823054,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00010739999999999998,
-      "loss": 6.910031127929687,
+      "epoch": 0.015166835187057633,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.0008950000000000001,
+      "loss": 4.4246673583984375,
       "step": 180
     },
     {
-      "epoch": 0.006740815638692282,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0001194,
-      "loss": 6.671029663085937,
+      "epoch": 0.016852039096730706,
+      "grad_norm": 0.87890625,
+      "learning_rate": 0.000995,
+      "loss": 4.204695129394532,
       "step": 200
     },
     {
-      "epoch": 0.006740815638692282,
-      "eval_loss": 6.548758506774902,
-      "eval_runtime": 8.6601,
-      "eval_samples_per_second": 1100.105,
-      "eval_steps_per_second": 0.808,
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.133424282073975,
+      "eval_runtime": 7.9329,
+      "eval_samples_per_second": 1200.942,
+      "eval_steps_per_second": 0.882,
       "step": 200
     },
     {
-      "epoch": 0.00741489720256151,
-      "grad_norm": 1.578125,
-      "learning_rate": 0.0001314,
-      "loss": 6.457804107666016,
+      "epoch": 0.018537243006403775,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 4.048733520507812,
       "step": 220
     },
     {
-      "epoch": 0.008088978766430738,
-      "grad_norm": 1.4921875,
-      "learning_rate": 0.0001434,
-      "loss": 6.254596328735351,
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 3.9190834045410154,
       "step": 240
     },
     {
-      "epoch": 0.008763060330299966,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00015539999999999998,
-      "loss": 6.047513961791992,
+      "epoch": 0.021907650825749917,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 3.7833847045898437,
       "step": 260
     },
     {
-      "epoch": 0.009437141894169195,
-      "grad_norm": 1.8828125,
-      "learning_rate": 0.0001674,
-      "loss": 5.870236587524414,
+      "epoch": 0.023592854735422986,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.700960159301758,
       "step": 280
     },
     {
-      "epoch": 0.010111223458038422,
-      "grad_norm": 1.359375,
-      "learning_rate": 0.00017939999999999997,
-      "loss": 5.72708740234375,
+      "epoch": 0.025278058645096056,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.001,
+      "loss": 3.6359439849853517,
       "step": 300
     },
     {
-      "epoch": 0.010111223458038422,
-      "eval_loss": 5.652859687805176,
-      "eval_runtime": 8.4927,
-      "eval_samples_per_second": 1121.794,
-      "eval_steps_per_second": 0.824,
+      "epoch": 0.025278058645096056,
+      "eval_loss": 3.5975606441497803,
+      "eval_runtime": 7.973,
+      "eval_samples_per_second": 1194.91,
+      "eval_steps_per_second": 0.878,
       "step": 300
     },
     {
-      "epoch": 0.010785305021907651,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0001914,
-      "loss": 5.585579681396484,
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.001,
+      "loss": 3.5393722534179686,
       "step": 320
     },
     {
-      "epoch": 0.011459386585776879,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.00020339999999999998,
-      "loss": 5.470914077758789,
+      "epoch": 0.028648466464442197,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.001,
+      "loss": 3.492219924926758,
       "step": 340
     },
     {
-      "epoch": 0.012133468149646108,
-      "grad_norm": 1.4140625,
-      "learning_rate": 0.00021539999999999998,
-      "loss": 5.318555068969727,
+      "epoch": 0.030333670374115267,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.4430694580078125,
       "step": 360
     },
     {
-      "epoch": 0.012807549713515335,
-      "grad_norm": 1.6015625,
-      "learning_rate": 0.00022739999999999997,
-      "loss": 5.185982894897461,
+      "epoch": 0.032018874283788336,
+      "grad_norm": 0.8203125,
+      "learning_rate": 0.001,
+      "loss": 3.3989883422851563,
       "step": 380
     },
     {
-      "epoch": 0.013481631277384564,
-      "grad_norm": 1.2578125,
-      "learning_rate": 0.0002394,
-      "loss": 5.065654754638672,
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 3.341664123535156,
       "step": 400
     },
     {
-      "epoch": 0.013481631277384564,
-      "eval_loss": 5.008047103881836,
-      "eval_runtime": 8.5503,
-      "eval_samples_per_second": 1114.232,
-      "eval_steps_per_second": 0.819,
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.3295202255249023,
+      "eval_runtime": 8.1687,
+      "eval_samples_per_second": 1166.287,
+      "eval_steps_per_second": 0.857,
       "step": 400
     },
     {
-      "epoch": 0.014155712841253791,
-      "grad_norm": 2.140625,
-      "learning_rate": 0.0002514,
-      "loss": 4.935818481445312,
+      "epoch": 0.03538928210313448,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 3.3047794342041015,
       "step": 420
     },
     {
-      "epoch": 0.01482979440512302,
-      "grad_norm": 1.1875,
-      "learning_rate": 0.00026339999999999995,
-      "loss": 4.853317642211914,
+      "epoch": 0.03707448601280755,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 3.258700942993164,
       "step": 440
     },
     {
-      "epoch": 0.015503875968992248,
-      "grad_norm": 1.5390625,
-      "learning_rate": 0.00027539999999999997,
-      "loss": 4.7492321014404295,
+      "epoch": 0.03875968992248062,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 3.229313278198242,
       "step": 460
     },
     {
-      "epoch": 0.016177957532861477,
-      "grad_norm": 1.921875,
-      "learning_rate": 0.00028739999999999994,
-      "loss": 4.629489898681641,
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.001,
+      "loss": 3.202067565917969,
       "step": 480
     },
     {
-      "epoch": 0.016852039096730706,
-      "grad_norm": 1.7890625,
-      "learning_rate": 0.00029939999999999996,
-      "loss": 4.565845108032226,
+      "epoch": 0.04213009774182676,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 3.163772201538086,
       "step": 500
     },
     {
-      "epoch": 0.016852039096730706,
-      "eval_loss": 4.530772686004639,
-      "eval_runtime": 8.4284,
-      "eval_samples_per_second": 1130.345,
-      "eval_steps_per_second": 0.831,
+      "epoch": 0.04213009774182676,
+      "eval_loss": 3.157355308532715,
+      "eval_runtime": 7.9503,
+      "eval_samples_per_second": 1198.315,
+      "eval_steps_per_second": 0.88,
       "step": 500
     },
     {
-      "epoch": 0.01752612066059993,
-      "grad_norm": 1.140625,
-      "learning_rate": 0.0003,
-      "loss": 4.50256233215332,
+      "epoch": 0.043815301651499834,
+      "grad_norm": 0.6953125,
+      "learning_rate": 0.001,
+      "loss": 3.1448848724365233,
       "step": 520
     },
     {
-      "epoch": 0.01820020222446916,
-      "grad_norm": 1.7265625,
-      "learning_rate": 0.0003,
-      "loss": 4.419484710693359,
+      "epoch": 0.0455005055611729,
+      "grad_norm": 0.828125,
+      "learning_rate": 0.001,
+      "loss": 3.103527069091797,
       "step": 540
     },
     {
-      "epoch": 0.01887428378833839,
-      "grad_norm": 1.3125,
-      "learning_rate": 0.0003,
-      "loss": 4.363323593139649,
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.001,
+      "loss": 3.08404541015625,
       "step": 560
     },
     {
-      "epoch": 0.01954836535220762,
-      "grad_norm": 1.5,
-      "learning_rate": 0.0003,
-      "loss": 4.3072765350341795,
+      "epoch": 0.04887091338051904,
+      "grad_norm": 0.76953125,
+      "learning_rate": 0.001,
+      "loss": 3.0501741409301757,
       "step": 580
     },
     {
-      "epoch": 0.020222446916076844,
-      "grad_norm": 1.9765625,
-      "learning_rate": 0.0003,
-      "loss": 4.263522338867188,
+      "epoch": 0.05055611729019211,
+      "grad_norm": 0.96875,
+      "learning_rate": 0.001,
+      "loss": 3.0376760482788088,
       "step": 600
     },
     {
-      "epoch": 0.020222446916076844,
-      "eval_loss": 4.239859580993652,
-      "eval_runtime": 8.5836,
-      "eval_samples_per_second": 1109.908,
-      "eval_steps_per_second": 0.816,
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.0310890674591064,
+      "eval_runtime": 8.1195,
+      "eval_samples_per_second": 1173.346,
+      "eval_steps_per_second": 0.862,
       "step": 600
     },
     {
-      "epoch": 0.020896528479946073,
-      "grad_norm": 1.2265625,
-      "learning_rate": 0.0003,
-      "loss": 4.207575988769531,
+      "epoch": 0.05224132119986518,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.001,
+      "loss": 3.0314205169677733,
       "step": 620
     },
     {
-      "epoch": 0.021570610043815303,
-      "grad_norm": 1.3359375,
-      "learning_rate": 0.0003,
-      "loss": 4.179453659057617,
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.640625,
+      "learning_rate": 0.001,
+      "loss": 3.0014928817749023,
       "step": 640
     },
     {
-      "epoch": 0.022244691607684528,
-      "grad_norm": 1.09375,
-      "learning_rate": 0.0003,
-      "loss": 4.13963508605957,
+      "epoch": 0.055611729019211326,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.001,
+      "loss": 2.9963293075561523,
       "step": 660
     },
     {
-      "epoch": 0.022918773171553757,
-      "grad_norm": 1.328125,
-      "learning_rate": 0.0003,
-      "loss": 4.0913646697998045,
+      "epoch": 0.057296932928884395,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 2.9568761825561523,
       "step": 680
     },
     {
-      "epoch": 0.023592854735422986,
-      "grad_norm": 1.609375,
-      "learning_rate": 0.0003,
-      "loss": 4.082810974121093,
+      "epoch": 0.058982136838557464,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.9395275115966797,
       "step": 700
     },
     {
-      "epoch": 0.023592854735422986,
-      "eval_loss": 4.061285495758057,
-      "eval_runtime": 8.3862,
-      "eval_samples_per_second": 1136.028,
-      "eval_steps_per_second": 0.835,
+      "epoch": 0.058982136838557464,
+      "eval_loss": 2.939429759979248,
+      "eval_runtime": 7.975,
+      "eval_samples_per_second": 1194.602,
+      "eval_steps_per_second": 0.878,
       "step": 700
     },
     {
-      "epoch": 0.024266936299292215,
-      "grad_norm": 1.453125,
-      "learning_rate": 0.0003,
-      "loss": 4.052593612670899,
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.001,
+      "loss": 2.922671890258789,
       "step": 720
     },
     {
-      "epoch": 0.02494101786316144,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.0003,
-      "loss": 4.038698196411133,
+      "epoch": 0.06235254465790361,
+      "grad_norm": 0.95703125,
+      "learning_rate": 0.001,
+      "loss": 2.9145633697509767,
       "step": 740
     },
     {
-      "epoch": 0.02561509942703067,
-      "grad_norm": 1.671875,
-      "learning_rate": 0.0003,
-      "loss": 3.99466552734375,
+      "epoch": 0.06403774856757667,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.001,
+      "loss": 2.8876668930053713,
       "step": 760
     },
     {
-      "epoch": 0.0262891809908999,
-      "grad_norm": 1.3671875,
-      "learning_rate": 0.0003,
-      "loss": 3.9530941009521485,
+      "epoch": 0.06572295247724974,
+      "grad_norm": 0.69140625,
+      "learning_rate": 0.001,
+      "loss": 2.8692466735839846,
       "step": 780
     },
     {
-      "epoch": 0.026963262554769128,
-      "grad_norm": 1.4375,
-      "learning_rate": 0.0003,
-      "loss": 3.95872802734375,
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.001,
+      "loss": 2.8788633346557617,
       "step": 800
     },
     {
-      "epoch": 0.026963262554769128,
-      "eval_loss": 3.9487836360931396,
-      "eval_runtime": 8.7643,
-      "eval_samples_per_second": 1087.026,
-      "eval_steps_per_second": 0.799,
+      "epoch": 0.06740815638692282,
+      "eval_loss": 2.8632190227508545,
+      "eval_runtime": 8.2773,
+      "eval_samples_per_second": 1150.973,
+      "eval_steps_per_second": 0.846,
       "step": 800
     },
     {
-      "epoch": 0.027637344118638354,
-      "grad_norm": 1.2734375,
-      "learning_rate": 0.0003,
-      "loss": 3.917556381225586,
+      "epoch": 0.0690933602965959,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.001,
+      "loss": 2.8496471405029298,
       "step": 820
     },
     {
-      "epoch": 0.028311425682507583,
-      "grad_norm": 1.390625,
-      "learning_rate": 0.0003,
-      "loss": 3.913285827636719,
+      "epoch": 0.07077856420626896,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.001,
+      "loss": 2.847636604309082,
       "step": 840
     },
     {
-      "epoch": 0.028985507246376812,
-      "grad_norm": 1.453125,
-      "learning_rate": 0.0003,
-      "loss": 3.9367324829101564,
+      "epoch": 0.07246376811594203,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 2.8339916229248048,
       "step": 860
     },
     {
-      "epoch": 0.02965958881024604,
-      "grad_norm": 1.328125,
-      "learning_rate": 0.0003,
-      "loss": 3.8899993896484375,
+      "epoch": 0.0741489720256151,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.8253028869628904,
       "step": 880
     },
     {
-      "epoch": 0.030333670374115267,
-      "grad_norm": 1.6875,
-      "learning_rate": 0.0003,
-      "loss": 3.8575370788574217,
+      "epoch": 0.07583417593528817,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.001,
+      "loss": 2.7853120803833007,
       "step": 900
     },
     {
-      "epoch": 0.030333670374115267,
-      "eval_loss": 3.871619701385498,
-      "eval_runtime": 8.7624,
-      "eval_samples_per_second": 1087.263,
-      "eval_steps_per_second": 0.799,
+      "epoch": 0.07583417593528817,
+      "eval_loss": 2.8034849166870117,
+      "eval_runtime": 7.9593,
+      "eval_samples_per_second": 1196.967,
+      "eval_steps_per_second": 0.879,
       "step": 900
     }
   ],
   "logging_steps": 20,
-  "max_steps": 2500,
+  "max_steps": 1000,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -414,8 +414,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 130674799411200.0,
-  "train_batch_size": 32,
+  "total_flos": 326686998528000.0,
+  "train_batch_size": 80,
   "trial_name": null,
   "trial_params": null
 }
diff --git a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/checkpoint-900/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/model.safetensors b/zain/Activation/out/mlp-linear-9L_run/model.safetensors
index db9a0d06ddce3f0331cea71838ce158e25a2a1ea..606564e71698655590c6ecadb0ca13273efa5cdc 100644
--- a/zain/Activation/out/mlp-linear-9L_run/model.safetensors
+++ b/zain/Activation/out/mlp-linear-9L_run/model.safetensors
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b2c45512c657af2390a8ad017958eac3003533c6b983fd882bc7a35e1a187ed
+oid sha256:9e9c917aa3b25bdbc40f473e1cf017022bafceb91187441f0ef9ae86b2f5e2e6
 size 4010544
diff --git a/zain/Activation/out/mlp-linear-9L_run/training_args.bin b/zain/Activation/out/mlp-linear-9L_run/training_args.bin
index 30ddff2ff9d7d91b6b4a6d29a26d1c56efce12e9..972defcddf189444df097fc66dcaaba45485f5ca 100644
--- a/zain/Activation/out/mlp-linear-9L_run/training_args.bin
+++ b/zain/Activation/out/mlp-linear-9L_run/training_args.bin
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2150980b46702de80cf5b9c95700c86dedab0889248b4672dd0d83fc71b3aa0
+oid sha256:1168ea4bbfb8182db6bef374718cd5e9bd631ffa3eb5aaea5cc2742de1e3c4e5
 size 4920
diff --git a/zain/Activation/out/mlp-linear-9L_run/training_log.jsonl b/zain/Activation/out/mlp-linear-9L_run/training_log.jsonl
index f7e77eeafce0ae92d93240a13b49ff63f6acad2f..e8147ba8231377483464d90efb5459d89c5d2381 100644
--- a/zain/Activation/out/mlp-linear-9L_run/training_log.jsonl
+++ b/zain/Activation/out/mlp-linear-9L_run/training_log.jsonl
@@ -1,152 +1,62 @@
-{"step": 20, "epoch": 0.0006740815638692282, "timestamp": 1787090024.3510127, "loss": 8.322640228271485, "grad_norm": 1.796875, "learning_rate": 1.14e-05, "train/total_time_seconds": 0.4871659614145756, "train/time_per_step_avg": 0.024358298070728777, "train/epoch_time_elapsed": 1.0523655116558075, "train/estimated_remaining_minutes": 1.0068096535901228}
-{"step": 40, "epoch": 0.0013481631277384564, "timestamp": 1787090025.2673562, "loss": 8.267462158203125, "grad_norm": 1.75, "learning_rate": 2.34e-05, "train/total_time_seconds": 0.9049943648278713, "train/time_per_step_avg": 0.022624859120696783, "train/epoch_time_elapsed": 1.9687108546495438, "train/estimated_remaining_minutes": 0.9276192239485681}
-{"step": 60, "epoch": 0.0020222446916076846, "timestamp": 1787090026.191169, "loss": 8.106219482421874, "grad_norm": 1.3046875, "learning_rate": 3.539999999999999e-05, "train/total_time_seconds": 1.3297606445848942, "train/time_per_step_avg": 0.022162677409748237, "train/epoch_time_elapsed": 2.8925236389040947, "train/estimated_remaining_minutes": 0.901282214663095}
-{"step": 80, "epoch": 0.002696326255476913, "timestamp": 1787090027.185015, "loss": 7.948130798339844, "grad_norm": 1.28125, "learning_rate": 4.7399999999999993e-05, "train/total_time_seconds": 1.776447206735611, "train/time_per_step_avg": 0.022205590084195138, "train/epoch_time_elapsed": 3.8863690681755543, "train/estimated_remaining_minutes": 0.8956254667292038}
-{"step": 100, "epoch": 0.003370407819346141, "timestamp": 1787090028.1093025, "loss": 7.777301788330078, "grad_norm": 1.25, "learning_rate": 5.94e-05, "train/total_time_seconds": 2.2005495578050613, "train/time_per_step_avg": 0.022005495578050614, "train/epoch_time_elapsed": 4.810656946152449, "train/estimated_remaining_minutes": 0.8802198231220245}
-{"step": 100, "epoch": 0.003370407819346141, "timestamp": 1787090036.6009336, "eval_loss": 7.679966926574707, "eval_runtime": 8.4901, "eval_samples_per_second": 1122.134, "eval_steps_per_second": 0.824, "train/total_time_seconds": 2.2005495578050613, "train/time_per_step_avg": 0.022005495578050614, "train/epoch_time_elapsed": 13.302287008613348, "train/estimated_remaining_minutes": 0.8802198231220245}
-{"step": 120, "epoch": 0.004044489383215369, "timestamp": 1787090037.5545268, "loss": 7.585383605957031, "grad_norm": 1.2578125, "learning_rate": 7.139999999999999e-05, "train/total_time_seconds": 2.619401503354311, "train/time_per_step_avg": 0.021322355419397355, "train/epoch_time_elapsed": 14.255881257355213, "train/estimated_remaining_minutes": 0.8658577191643417}
-{"step": 140, "epoch": 0.0047185709470845974, "timestamp": 1787090038.4570768, "loss": 7.3637535095214846, "grad_norm": 1.25, "learning_rate": 8.34e-05, "train/total_time_seconds": 3.0328244753181934, "train/time_per_step_avg": 0.02127830110490322, "train/epoch_time_elapsed": 15.15843179449439, "train/estimated_remaining_minutes": 0.852079257351302}
-{"step": 160, "epoch": 0.005392652510953826, "timestamp": 1787090039.3583179, "loss": 7.1375579833984375, "grad_norm": 1.25, "learning_rate": 9.539999999999999e-05, "train/total_time_seconds": 3.4454963505268097, "train/time_per_step_avg": 0.021157357059419155, "train/epoch_time_elapsed": 16.059672601521015, "train/estimated_remaining_minutes": 0.8398397354409098}
-{"step": 180, "epoch": 0.006066734074823054, "timestamp": 1787090040.2629287, "loss": 6.910031127929687, "grad_norm": 1.34375, "learning_rate": 0.00010739999999999998, "train/total_time_seconds": 3.861355599015951, "train/time_per_step_avg": 0.0208490839228034, "train/epoch_time_elapsed": 16.964283030480146, "train/estimated_remaining_minutes": 0.8294763879367599}
-{"step": 200, "epoch": 0.006740815638692282, "timestamp": 1787090041.1701689, "loss": 6.671029663085937, "grad_norm": 1.3359375, "learning_rate": 0.0001194, "train/total_time_seconds": 4.279542360454798, "train/time_per_step_avg": 0.020789928026497363, "train/epoch_time_elapsed": 17.871523462235928, "train/estimated_remaining_minutes": 0.8202456190871695}
-{"step": 200, "epoch": 0.006740815638692282, "timestamp": 1787090049.8318973, "eval_loss": 6.548758506774902, "eval_runtime": 8.6601, "eval_samples_per_second": 1100.105, "eval_steps_per_second": 0.808, "train/total_time_seconds": 4.279542360454798, "train/time_per_step_avg": 0.020789928026497363, "train/epoch_time_elapsed": 26.53324942290783, "train/estimated_remaining_minutes": 0.8202456190871695}
-{"step": 220, "epoch": 0.00741489720256151, "timestamp": 1787090050.7864394, "loss": 6.457804107666016, "grad_norm": 1.578125, "learning_rate": 0.0001314, "train/total_time_seconds": 4.701243340969086, "train/time_per_step_avg": 0.020818418376147747, "train/epoch_time_elapsed": 27.48779420554638, "train/estimated_remaining_minutes": 0.812032940712842}
-{"step": 240, "epoch": 0.008088978766430738, "timestamp": 1787090051.7082744, "loss": 6.254596328735351, "grad_norm": 1.4921875, "learning_rate": 0.0001434, "train/total_time_seconds": 5.123800829052925, "train/time_per_step_avg": 0.020909763537347317, "train/epoch_time_elapsed": 28.40962889045477, "train/estimated_remaining_minutes": 0.8041520745596952}
-{"step": 260, "epoch": 0.008763060330299966, "timestamp": 1787090052.6360495, "loss": 6.047513961791992, "grad_norm": 1.171875, "learning_rate": 0.00015539999999999998, "train/total_time_seconds": 5.546675357967615, "train/time_per_step_avg": 0.021011790074408055, "train/epoch_time_elapsed": 29.337404031306505, "train/estimated_remaining_minutes": 0.7964456924261191}
-{"step": 280, "epoch": 0.009437141894169195, "timestamp": 1787090053.5612154, "loss": 5.870236587524414, "grad_norm": 1.8828125, "learning_rate": 0.0001674, "train/total_time_seconds": 5.969429299235344, "train/time_per_step_avg": 0.021080737002193928, "train/epoch_time_elapsed": 30.262570057064295, "train/estimated_remaining_minutes": 0.7888174431132419}
-{"step": 300, "epoch": 0.010111223458038422, "timestamp": 1787090054.5063047, "loss": 5.72708740234375, "grad_norm": 1.359375, "learning_rate": 0.00017939999999999997, "train/total_time_seconds": 6.396400835365057, "train/time_per_step_avg": 0.021168584749102593, "train/epoch_time_elapsed": 31.207659542560577, "train/estimated_remaining_minutes": 0.7817823243223958}
-{"step": 300, "epoch": 0.010111223458038422, "timestamp": 1787090063.000556, "eval_loss": 5.652859687805176, "eval_runtime": 8.4927, "eval_samples_per_second": 1121.794, "eval_steps_per_second": 0.824, "train/total_time_seconds": 6.396400835365057, "train/time_per_step_avg": 0.021168584749102593, "train/epoch_time_elapsed": 39.70190812647343, "train/estimated_remaining_minutes": 0.7817823243223958}
-{"step": 320, "epoch": 0.010785305021907651, "timestamp": 1787090064.0466053, "loss": 5.585579681396484, "grad_norm": 1.609375, "learning_rate": 0.0001914, "train/total_time_seconds": 6.901587720960379, "train/time_per_step_avg": 0.02200344379991293, "train/epoch_time_elapsed": 40.74796038493514, "train/estimated_remaining_minutes": 0.783617772484043}
-{"step": 340, "epoch": 0.011459386585776879, "timestamp": 1787090065.0750475, "loss": 5.470914077758789, "grad_norm": 1.671875, "learning_rate": 0.00020339999999999998, "train/total_time_seconds": 7.3942382000386715, "train/time_per_step_avg": 0.022704373709857464, "train/epoch_time_elapsed": 41.776401951909065, "train/estimated_remaining_minutes": 0.782919338827624}
-{"step": 360, "epoch": 0.012133468149646108, "timestamp": 1787090066.010938, "loss": 5.318555068969727, "grad_norm": 1.4140625, "learning_rate": 0.00021539999999999998, "train/total_time_seconds": 7.831252176314592, "train/time_per_step_avg": 0.022845768183469773, "train/epoch_time_elapsed": 42.71229239925742, "train/estimated_remaining_minutes": 0.7758740582089457}
-{"step": 380, "epoch": 0.012807549713515335, "timestamp": 1787090066.942198, "loss": 5.185982894897461, "grad_norm": 1.6015625, "learning_rate": 0.00022739999999999997, "train/total_time_seconds": 8.253654949367046, "train/time_per_step_avg": 0.022842256501317024, "train/epoch_time_elapsed": 43.643552385270596, "train/estimated_remaining_minutes": 0.7674451093271114}
-{"step": 400, "epoch": 0.013481631277384564, "timestamp": 1787090067.867534, "loss": 5.065654754638672, "grad_norm": 1.2578125, "learning_rate": 0.0002394, "train/total_time_seconds": 8.672141395509243, "train/time_per_step_avg": 0.02275740560144186, "train/epoch_time_elapsed": 44.56888834387064, "train/estimated_remaining_minutes": 0.7588123721070588}
-{"step": 400, "epoch": 0.013481631277384564, "timestamp": 1787090076.4197314, "eval_loss": 5.008047103881836, "eval_runtime": 8.5503, "eval_samples_per_second": 1114.232, "eval_steps_per_second": 0.819, "train/total_time_seconds": 8.672141395509243, "train/time_per_step_avg": 0.02275740560144186, "train/epoch_time_elapsed": 53.12108425050974, "train/estimated_remaining_minutes": 0.7588123721070588}
-{"step": 420, "epoch": 0.014155712841253791, "timestamp": 1787090077.4167783, "loss": 4.935818481445312, "grad_norm": 2.140625, "learning_rate": 0.0002514, "train/total_time_seconds": 9.099947553128004, "train/time_per_step_avg": 0.021983598321676255, "train/epoch_time_elapsed": 54.11813282221556, "train/estimated_remaining_minutes": 0.7511067821629464}
-{"step": 440, "epoch": 0.01482979440512302, "timestamp": 1787090078.3350494, "loss": 4.853317642211914, "grad_norm": 1.1875, "learning_rate": 0.00026339999999999995, "train/total_time_seconds": 9.511507794260979, "train/time_per_step_avg": 0.021172695942223072, "train/epoch_time_elapsed": 55.03640407696366, "train/estimated_remaining_minutes": 0.7421858354612733}
-{"step": 460, "epoch": 0.015503875968992248, "timestamp": 1787090079.2550533, "loss": 4.7492321014404295, "grad_norm": 1.5390625, "learning_rate": 0.00027539999999999997, "train/total_time_seconds": 9.924589660018682, "train/time_per_step_avg": 0.0209333748370409, "train/epoch_time_elapsed": 55.95640867948532, "train/estimated_remaining_minutes": 0.7335566270448591}
-{"step": 480, "epoch": 0.016177957532861477, "timestamp": 1787090080.1738203, "loss": 4.629489898681641, "grad_norm": 1.921875, "learning_rate": 0.00028739999999999994, "train/total_time_seconds": 10.338026851415634, "train/time_per_step_avg": 0.020843719020485877, "train/epoch_time_elapsed": 56.8751747533679, "train/estimated_remaining_minutes": 0.725097716661791}
-{"step": 500, "epoch": 0.016852039096730706, "timestamp": 1787090081.1055214, "loss": 4.565845108032226, "grad_norm": 1.7890625, "learning_rate": 0.00029939999999999996, "train/total_time_seconds": 10.760399051010609, "train/time_per_step_avg": 0.020882576555013656, "train/epoch_time_elapsed": 57.80687604099512, "train/estimated_remaining_minutes": 0.7173599367340405}
-{"step": 500, "epoch": 0.016852039096730706, "timestamp": 1787090089.5357459, "eval_loss": 4.530772686004639, "eval_runtime": 8.4284, "eval_samples_per_second": 1130.345, "eval_steps_per_second": 0.831, "train/total_time_seconds": 10.760399051010609, "train/time_per_step_avg": 0.020882576555013656, "train/epoch_time_elapsed": 66.23709952831268, "train/estimated_remaining_minutes": 0.7173599367340405}
-{"step": 520, "epoch": 0.01752612066059993, "timestamp": 1787090090.4781623, "loss": 4.50256233215332, "grad_norm": 1.140625, "learning_rate": 0.0003, "train/total_time_seconds": 11.175844598561525, "train/time_per_step_avg": 0.020758970454335213, "train/epoch_time_elapsed": 67.17951717972755, "train/estimated_remaining_minutes": 0.709236291831789}
-{"step": 540, "epoch": 0.01820020222446916, "timestamp": 1787090091.3960457, "loss": 4.419484710693359, "grad_norm": 1.7265625, "learning_rate": 0.0003, "train/total_time_seconds": 11.595506254583597, "train/time_per_step_avg": 0.020839984603226183, "train/epoch_time_elapsed": 68.09740042313933, "train/estimated_remaining_minutes": 0.7014565512032054}
-{"step": 560, "epoch": 0.01887428378833839, "timestamp": 1787090092.3256834, "loss": 4.363323593139649, "grad_norm": 1.3125, "learning_rate": 0.0003, "train/total_time_seconds": 12.02891132235527, "train/time_per_step_avg": 0.02104321662336588, "train/epoch_time_elapsed": 69.02703780308366, "train/estimated_remaining_minutes": 0.6945264275407507}
-{"step": 580, "epoch": 0.01954836535220762, "timestamp": 1787090093.2448857, "loss": 4.3072765350341795, "grad_norm": 1.5, "learning_rate": 0.0003, "train/total_time_seconds": 12.451076030731201, "train/time_per_step_avg": 0.02113049179315567, "train/epoch_time_elapsed": 69.94624043628573, "train/estimated_remaining_minutes": 0.6869559189368939}
-{"step": 600, "epoch": 0.020222446916076844, "timestamp": 1787090094.1727655, "loss": 4.263522338867188, "grad_norm": 1.9765625, "learning_rate": 0.0003, "train/total_time_seconds": 12.882123444229364, "train/time_per_step_avg": 0.021217243932187557, "train/epoch_time_elapsed": 70.87412008270621, "train/estimated_remaining_minutes": 0.6798898484454388}
-{"step": 600, "epoch": 0.020222446916076844, "timestamp": 1787090102.7580671, "eval_loss": 4.239859580993652, "eval_runtime": 8.5836, "eval_samples_per_second": 1109.908, "eval_steps_per_second": 0.816, "train/total_time_seconds": 12.882123444229364, "train/time_per_step_avg": 0.021217243932187557, "train/epoch_time_elapsed": 79.45942096784711, "train/estimated_remaining_minutes": 0.6798898484454388}
-{"step": 620, "epoch": 0.020896528479946073, "timestamp": 1787090103.7138457, "loss": 4.207575988769531, "grad_norm": 1.2265625, "learning_rate": 0.0003, "train/total_time_seconds": 13.297651916742325, "train/time_per_step_avg": 0.021218073181807996, "train/epoch_time_elapsed": 80.41520067676902, "train/estimated_remaining_minutes": 0.6720318710611711}
-{"step": 640, "epoch": 0.021570610043815303, "timestamp": 1787090104.6780584, "loss": 4.179453659057617, "grad_norm": 1.3359375, "learning_rate": 0.0003, "train/total_time_seconds": 13.743419598788023, "train/time_per_step_avg": 0.02147913344204426, "train/epoch_time_elapsed": 81.37941356003284, "train/estimated_remaining_minutes": 0.6656968868162949}
-{"step": 660, "epoch": 0.022244691607684528, "timestamp": 1787090105.6800876, "loss": 4.13963508605957, "grad_norm": 1.09375, "learning_rate": 0.0003, "train/total_time_seconds": 14.178391981869936, "train/time_per_step_avg": 0.021494806595146656, "train/epoch_time_elapsed": 82.38144203275442, "train/estimated_remaining_minutes": 0.6587939708747647}
-{"step": 680, "epoch": 0.022918773171553757, "timestamp": 1787090106.6163764, "loss": 4.0913646697998045, "grad_norm": 1.328125, "learning_rate": 0.0003, "train/total_time_seconds": 14.603615425527096, "train/time_per_step_avg": 0.021525393947958948, "train/epoch_time_elapsed": 83.31773166358471, "train/estimated_remaining_minutes": 0.6514357861387087}
-{"step": 700, "epoch": 0.023592854735422986, "timestamp": 1787090107.6156392, "loss": 4.082810974121093, "grad_norm": 1.609375, "learning_rate": 0.0003, "train/total_time_seconds": 15.048711270093918, "train/time_per_step_avg": 0.021665878258645535, "train/epoch_time_elapsed": 84.31699389219284, "train/estimated_remaining_minutes": 0.6449447687183107}
-{"step": 700, "epoch": 0.023592854735422986, "timestamp": 1787090116.0037506, "eval_loss": 4.061285495758057, "eval_runtime": 8.3862, "eval_samples_per_second": 1136.028, "eval_steps_per_second": 0.835, "train/total_time_seconds": 15.048711270093918, "train/time_per_step_avg": 0.021665878258645535, "train/epoch_time_elapsed": 92.7051047757268, "train/estimated_remaining_minutes": 0.6449447687183107}
-{"step": 720, "epoch": 0.024266936299292215, "timestamp": 1787090116.9622593, "loss": 4.052593612670899, "grad_norm": 1.453125, "learning_rate": 0.0003, "train/total_time_seconds": 15.459270983934402, "train/time_per_step_avg": 0.021616190671920776, "train/epoch_time_elapsed": 93.66361425817013, "train/estimated_remaining_minutes": 0.6369792210972971}
-{"step": 740, "epoch": 0.02494101786316144, "timestamp": 1787090117.886146, "loss": 4.038698196411133, "grad_norm": 1.46875, "learning_rate": 0.0003, "train/total_time_seconds": 15.874074883759022, "train/time_per_step_avg": 0.02130655284970999, "train/epoch_time_elapsed": 94.5875009149313, "train/estimated_remaining_minutes": 0.6292426080048621}
-{"step": 760, "epoch": 0.02561509942703067, "timestamp": 1787090118.8126843, "loss": 3.99466552734375, "grad_norm": 1.671875, "learning_rate": 0.0003, "train/total_time_seconds": 16.288574557751417, "train/time_per_step_avg": 0.02110182575881481, "train/epoch_time_elapsed": 95.51403892412782, "train/estimated_remaining_minutes": 0.6215377133878831}
-{"step": 780, "epoch": 0.0262891809908999, "timestamp": 1787090119.7369952, "loss": 3.9530941009521485, "grad_norm": 1.3671875, "learning_rate": 0.0003, "train/total_time_seconds": 16.70217404142022, "train/time_per_step_avg": 0.020985586158931254, "train/epoch_time_elapsed": 96.43835016340017, "train/estimated_remaining_minutes": 0.6138405844282645}
-{"step": 800, "epoch": 0.026963262554769128, "timestamp": 1787090120.6631215, "loss": 3.95872802734375, "grad_norm": 1.4375, "learning_rate": 0.0003, "train/total_time_seconds": 17.1165285743773, "train/time_per_step_avg": 0.020678173042833804, "train/epoch_time_elapsed": 97.36447604000568, "train/estimated_remaining_minutes": 0.6062103870091959}
-{"step": 800, "epoch": 0.026963262554769128, "timestamp": 1787090129.4290054, "eval_loss": 3.9487836360931396, "eval_runtime": 8.7643, "eval_samples_per_second": 1087.026, "eval_steps_per_second": 0.799, "train/total_time_seconds": 17.1165285743773, "train/time_per_step_avg": 0.020678173042833804, "train/epoch_time_elapsed": 106.13035868108273, "train/estimated_remaining_minutes": 0.6062103870091959}
-{"step": 820, "epoch": 0.027637344118638354, "timestamp": 1787090130.3672554, "loss": 3.917556381225586, "grad_norm": 1.2734375, "learning_rate": 0.0003, "train/total_time_seconds": 17.527744065970182, "train/time_per_step_avg": 0.0206847308203578, "train/epoch_time_elapsed": 107.06861016899347, "train/estimated_remaining_minutes": 0.5985083339599574}
-{"step": 840, "epoch": 0.028311425682507583, "timestamp": 1787090131.2712123, "loss": 3.913285827636719, "grad_norm": 1.390625, "learning_rate": 0.0003, "train/total_time_seconds": 17.939896412193775, "train/time_per_step_avg": 0.020658215284347536, "train/epoch_time_elapsed": 107.97256690636277, "train/estimated_remaining_minutes": 0.5908775405603506}
-{"step": 860, "epoch": 0.028985507246376812, "timestamp": 1787090132.1687262, "loss": 3.9367324829101564, "grad_norm": 1.453125, "learning_rate": 0.0003, "train/total_time_seconds": 18.34703105315566, "train/time_per_step_avg": 0.020584564954042435, "train/epoch_time_elapsed": 108.87008076161146, "train/estimated_remaining_minutes": 0.5831226923871179}
-{"step": 880, "epoch": 0.02965958881024604, "timestamp": 1787090133.0716784, "loss": 3.8899993896484375, "grad_norm": 1.328125, "learning_rate": 0.0003, "train/total_time_seconds": 18.755650278180838, "train/time_per_step_avg": 0.020534762367606162, "train/epoch_time_elapsed": 109.77303267642856, "train/estimated_remaining_minutes": 0.5754574517169121}
-{"step": 900, "epoch": 0.030333670374115267, "timestamp": 1787090133.972401, "loss": 3.8575370788574217, "grad_norm": 1.6875, "learning_rate": 0.0003, "train/total_time_seconds": 19.16444643959403, "train/time_per_step_avg": 0.02047917865216732, "train/epoch_time_elapsed": 110.67375592142344, "train/estimated_remaining_minutes": 0.5678354500620454}
-{"step": 900, "epoch": 0.030333670374115267, "timestamp": 1787090142.736271, "eval_loss": 3.871619701385498, "eval_runtime": 8.7624, "eval_samples_per_second": 1087.263, "eval_steps_per_second": 0.799, "train/total_time_seconds": 19.16444643959403, "train/time_per_step_avg": 0.02047917865216732, "train/epoch_time_elapsed": 119.43762295693159, "train/estimated_remaining_minutes": 0.5678354500620454}
-{"step": 920, "epoch": 0.031007751937984496, "timestamp": 1787090143.6765184, "loss": 3.870378112792969, "grad_norm": 1.1875, "learning_rate": 0.0003, "train/total_time_seconds": 19.576474107801914, "train/time_per_step_avg": 0.020487300418317318, "train/epoch_time_elapsed": 120.37787260860205, "train/estimated_remaining_minutes": 0.5603411067088229}
-{"step": 940, "epoch": 0.031681833501853725, "timestamp": 1787090144.648713, "loss": 3.852782440185547, "grad_norm": 1.234375, "learning_rate": 0.0003, "train/total_time_seconds": 20.05193056166172, "train/time_per_step_avg": 0.02112034149467945, "train/epoch_time_elapsed": 121.3500671684742, "train/estimated_remaining_minutes": 0.554627866599154}
-{"step": 960, "epoch": 0.032355915065722954, "timestamp": 1787090145.5583317, "loss": 3.821004867553711, "grad_norm": 1.515625, "learning_rate": 0.0003, "train/total_time_seconds": 20.465665958821774, "train/time_per_step_avg": 0.021186349056661127, "train/epoch_time_elapsed": 122.25968647748232, "train/estimated_remaining_minutes": 0.5471723190379432}
-{"step": 980, "epoch": 0.03302999662959218, "timestamp": 1787090146.4644017, "loss": 3.8098331451416017, "grad_norm": 1.4921875, "learning_rate": 0.0003, "train/total_time_seconds": 20.878857355564833, "train/time_per_step_avg": 0.02123207077383995, "train/epoch_time_elapsed": 123.16575583443046, "train/estimated_remaining_minutes": 0.5397255642935126}
-{"step": 1000, "epoch": 0.03370407819346141, "timestamp": 1787090147.4082043, "loss": 3.806192398071289, "grad_norm": 1.4453125, "learning_rate": 0.0003, "train/total_time_seconds": 21.309228148311377, "train/time_per_step_avg": 0.02144781708717346, "train/epoch_time_elapsed": 124.10955806076527, "train/estimated_remaining_minutes": 0.5327307037077844}
-{"step": 1000, "epoch": 0.03370407819346141, "timestamp": 1787090155.9603746, "eval_loss": 3.805492401123047, "eval_runtime": 8.55, "eval_samples_per_second": 1114.265, "eval_steps_per_second": 0.819, "train/total_time_seconds": 21.309228148311377, "train/time_per_step_avg": 0.02144781708717346, "train/epoch_time_elapsed": 132.6617270372808, "train/estimated_remaining_minutes": 0.5327307037077844}
-{"step": 1020, "epoch": 0.034378159757330634, "timestamp": 1787090156.9198642, "loss": 3.7934722900390625, "grad_norm": 1.375, "learning_rate": 0.0003, "train/total_time_seconds": 21.73783740401268, "train/time_per_step_avg": 0.021613632962107658, "train/epoch_time_elapsed": 133.6212188899517, "train/estimated_remaining_minutes": 0.5256862640186073}
-{"step": 1040, "epoch": 0.03505224132119986, "timestamp": 1787090157.882276, "loss": 3.773797607421875, "grad_norm": 1.3515625, "learning_rate": 0.0003, "train/total_time_seconds": 22.17101999744773, "train/time_per_step_avg": 0.021190894357860087, "train/epoch_time_elapsed": 134.58363071084023, "train/estimated_remaining_minutes": 0.5187450191710526}
-{"step": 1060, "epoch": 0.03572632288506909, "timestamp": 1787090158.801586, "loss": 3.7539413452148436, "grad_norm": 1.734375, "learning_rate": 0.0003, "train/total_time_seconds": 22.594436164945364, "train/time_per_step_avg": 0.021287702061235904, "train/epoch_time_elapsed": 135.50294019281864, "train/estimated_remaining_minutes": 0.5115721395836687}
-{"step": 1080, "epoch": 0.03640040444893832, "timestamp": 1787090159.7369173, "loss": 3.742329406738281, "grad_norm": 2.109375, "learning_rate": 0.0003, "train/total_time_seconds": 23.02944030240178, "train/time_per_step_avg": 0.021505829468369483, "train/epoch_time_elapsed": 136.4382721632719, "train/estimated_remaining_minutes": 0.5046574881081871}
-{"step": 1100, "epoch": 0.03707448601280755, "timestamp": 1787090160.6459656, "loss": 3.737290954589844, "grad_norm": 1.84375, "learning_rate": 0.0003, "train/total_time_seconds": 23.445339139550924, "train/time_per_step_avg": 0.02136110991239548, "train/epoch_time_elapsed": 137.34732024744153, "train/estimated_remaining_minutes": 0.49732537568744384}
-{"step": 1100, "epoch": 0.03707448601280755, "timestamp": 1787090169.0898743, "eval_loss": 3.744091272354126, "eval_runtime": 8.4423, "eval_samples_per_second": 1128.49, "eval_steps_per_second": 0.829, "train/total_time_seconds": 23.445339139550924, "train/time_per_step_avg": 0.02136110991239548, "train/epoch_time_elapsed": 145.79122596606612, "train/estimated_remaining_minutes": 0.49732537568744384}
-{"step": 1120, "epoch": 0.03774856757667678, "timestamp": 1787090170.1453664, "loss": 3.7377304077148437, "grad_norm": 1.2890625, "learning_rate": 0.0003, "train/total_time_seconds": 23.914364136755466, "train/time_per_step_avg": 0.021765267327427864, "train/epoch_time_elapsed": 146.84672116488218, "train/estimated_remaining_minutes": 0.49109854923694257}
-{"step": 1140, "epoch": 0.03842264914054601, "timestamp": 1787090171.1464329, "loss": 3.713370513916016, "grad_norm": 1.9140625, "learning_rate": 0.0003, "train/total_time_seconds": 24.421343587338924, "train/time_per_step_avg": 0.02250323589891195, "train/epoch_time_elapsed": 147.8477869592607, "train/estimated_remaining_minutes": 0.4855705742511833}
-{"step": 1160, "epoch": 0.03909673070441524, "timestamp": 1787090172.061697, "loss": 3.7126068115234374, "grad_norm": 1.703125, "learning_rate": 0.0003, "train/total_time_seconds": 24.84479048475623, "train/time_per_step_avg": 0.022503543198108673, "train/epoch_time_elapsed": 148.76305164396763, "train/estimated_remaining_minutes": 0.4783336099076631}
-{"step": 1180, "epoch": 0.03977081226828446, "timestamp": 1787090172.96644, "loss": 3.7255340576171876, "grad_norm": 1.3515625, "learning_rate": 0.0003, "train/total_time_seconds": 25.258656304329634, "train/time_per_step_avg": 0.022292160019278525, "train/epoch_time_elapsed": 149.66779493540525, "train/estimated_remaining_minutes": 0.47092410058919654}
-{"step": 1200, "epoch": 0.04044489383215369, "timestamp": 1787090173.8752575, "loss": 3.6966705322265625, "grad_norm": 1.4140625, "learning_rate": 0.0003, "train/total_time_seconds": 25.67636000365019, "train/time_per_step_avg": 0.022310208640992642, "train/epoch_time_elapsed": 150.57661206647754, "train/estimated_remaining_minutes": 0.4636009445103506}
-{"step": 1200, "epoch": 0.04044489383215369, "timestamp": 1787090182.3466508, "eval_loss": 3.6958041191101074, "eval_runtime": 8.4698, "eval_samples_per_second": 1124.816, "eval_steps_per_second": 0.826, "train/total_time_seconds": 25.67636000365019, "train/time_per_step_avg": 0.022310208640992642, "train/epoch_time_elapsed": 159.04800394922495, "train/estimated_remaining_minutes": 0.4636009445103506}
-{"step": 1220, "epoch": 0.04111897539602292, "timestamp": 1787090183.3028944, "loss": 3.6724082946777346, "grad_norm": 1.6171875, "learning_rate": 0.0003, "train/total_time_seconds": 26.09580424427986, "train/time_per_step_avg": 0.02181440107524395, "train/epoch_time_elapsed": 160.0042488835752, "train/estimated_remaining_minutes": 0.45632007421691567}
-{"step": 1240, "epoch": 0.04179305695989215, "timestamp": 1787090184.2335675, "loss": 3.6844207763671877, "grad_norm": 1.5234375, "learning_rate": 0.0003, "train/total_time_seconds": 26.528886377811432, "train/time_per_step_avg": 0.021075427904725073, "train/epoch_time_elapsed": 160.9349226243794, "train/estimated_remaining_minutes": 0.44927952736616134}
-{"step": 1260, "epoch": 0.042467138523761376, "timestamp": 1787090185.199458, "loss": 3.6926769256591796, "grad_norm": 1.375, "learning_rate": 0.0003, "train/total_time_seconds": 26.96605222299695, "train/time_per_step_avg": 0.02121261738240719, "train/epoch_time_elapsed": 161.90081299096346, "train/estimated_remaining_minutes": 0.4423003274671457}
-{"step": 1280, "epoch": 0.043141220087630605, "timestamp": 1787090186.1345897, "loss": 3.6725536346435548, "grad_norm": 1.5078125, "learning_rate": 0.0003, "train/total_time_seconds": 27.402600783854723, "train/time_per_step_avg": 0.02143944479525089, "train/epoch_time_elapsed": 162.83594417572021, "train/estimated_remaining_minutes": 0.4353017312018589}
-{"step": 1300, "epoch": 0.043815301651499834, "timestamp": 1787090187.0789473, "loss": 3.6652542114257813, "grad_norm": 1.4921875, "learning_rate": 0.0003, "train/total_time_seconds": 27.839303817600012, "train/time_per_step_avg": 0.021629438139498233, "train/epoch_time_elapsed": 163.78030264377594, "train/estimated_remaining_minutes": 0.4282969818092309}
-{"step": 1300, "epoch": 0.043815301651499834, "timestamp": 1787090195.647478, "eval_loss": 3.6601669788360596, "eval_runtime": 8.567, "eval_samples_per_second": 1112.052, "eval_steps_per_second": 0.817, "train/total_time_seconds": 27.839303817600012, "train/time_per_step_avg": 0.021629438139498233, "train/epoch_time_elapsed": 172.34883080422878, "train/estimated_remaining_minutes": 0.4282969818092309}
-{"step": 1320, "epoch": 0.044489383215369056, "timestamp": 1787090196.6083207, "loss": 3.6304325103759765, "grad_norm": 1.203125, "learning_rate": 0.0003, "train/total_time_seconds": 28.265503082424402, "train/time_per_step_avg": 0.021696988381445407, "train/epoch_time_elapsed": 173.3096746765077, "train/estimated_remaining_minutes": 0.421127444914909}
-{"step": 1340, "epoch": 0.045163464779238285, "timestamp": 1787090197.5520518, "loss": 3.654827880859375, "grad_norm": 1.328125, "learning_rate": 0.0003, "train/total_time_seconds": 28.698285780847073, "train/time_per_step_avg": 0.021693994030356406, "train/epoch_time_elapsed": 174.25340588018298, "train/estimated_remaining_minutes": 0.414054869474908}
-{"step": 1360, "epoch": 0.045837546343107514, "timestamp": 1787090198.5060043, "loss": 3.634907531738281, "grad_norm": 1.3671875, "learning_rate": 0.0003, "train/total_time_seconds": 29.14553214609623, "train/time_per_step_avg": 0.021794799230992794, "train/epoch_time_elapsed": 175.20735903084278, "train/estimated_remaining_minutes": 0.4071802285116385}
-{"step": 1380, "epoch": 0.046511627906976744, "timestamp": 1787090199.4406104, "loss": 3.631271743774414, "grad_norm": 1.3203125, "learning_rate": 0.0003, "train/total_time_seconds": 29.57575212791562, "train/time_per_step_avg": 0.021731513440608977, "train/epoch_time_elapsed": 176.14196480065584, "train/estimated_remaining_minutes": 0.4000584828896799}
-{"step": 1400, "epoch": 0.04718570947084597, "timestamp": 1787090200.3831522, "loss": 3.6088893890380858, "grad_norm": 1.4765625, "learning_rate": 0.0003, "train/total_time_seconds": 30.00523616373539, "train/time_per_step_avg": 0.02165932346135378, "train/epoch_time_elapsed": 177.08450701460242, "train/estimated_remaining_minutes": 0.3929257116679634}
-{"step": 1400, "epoch": 0.04718570947084597, "timestamp": 1787090208.7806206, "eval_loss": 3.620887517929077, "eval_runtime": 8.3958, "eval_samples_per_second": 1134.733, "eval_steps_per_second": 0.834, "train/total_time_seconds": 30.00523616373539, "train/time_per_step_avg": 0.02165932346135378, "train/epoch_time_elapsed": 185.48197343945503, "train/estimated_remaining_minutes": 0.3929257116679634}
-{"step": 1420, "epoch": 0.0478597910347152, "timestamp": 1787090209.758754, "loss": 3.6118431091308594, "grad_norm": 1.3125, "learning_rate": 0.0003, "train/total_time_seconds": 30.44477339833975, "train/time_per_step_avg": 0.02179270315915346, "train/epoch_time_elapsed": 186.46010866761208, "train/estimated_remaining_minutes": 0.385919662795856}
-{"step": 1440, "epoch": 0.04853387259858443, "timestamp": 1787090210.6889102, "loss": 3.6007781982421876, "grad_norm": 1.4140625, "learning_rate": 0.0003, "train/total_time_seconds": 30.87500672414899, "train/time_per_step_avg": 0.02176720943301916, "train/epoch_time_elapsed": 187.390264775604, "train/estimated_remaining_minutes": 0.37879059175460567}
-{"step": 1460, "epoch": 0.04920795416245366, "timestamp": 1787090211.629568, "loss": 3.577118682861328, "grad_norm": 1.171875, "learning_rate": 0.0003, "train/total_time_seconds": 31.31340730935335, "train/time_per_step_avg": 0.02167875163257122, "train/epoch_time_elapsed": 188.33092243596911, "train/estimated_remaining_minutes": 0.3717573470516837}
-{"step": 1480, "epoch": 0.04988203572632288, "timestamp": 1787090212.5728993, "loss": 3.600400924682617, "grad_norm": 1.453125, "learning_rate": 0.0003, "train/total_time_seconds": 31.75088044628501, "train/time_per_step_avg": 0.021751283183693886, "train/epoch_time_elapsed": 189.27425381541252, "train/estimated_remaining_minutes": 0.3647060591803008}
-{"step": 1500, "epoch": 0.05055611729019211, "timestamp": 1787090213.5248115, "loss": 3.5930843353271484, "grad_norm": 1.3984375, "learning_rate": 0.0003, "train/total_time_seconds": 32.199186488986015, "train/time_per_step_avg": 0.021939503252506255, "train/epoch_time_elapsed": 190.22616611793637, "train/estimated_remaining_minutes": 0.3577687387665113}
-{"step": 1500, "epoch": 0.05055611729019211, "timestamp": 1787090222.1475577, "eval_loss": 3.5886731147766113, "eval_runtime": 8.6212, "eval_samples_per_second": 1105.07, "eval_steps_per_second": 0.812, "train/total_time_seconds": 32.199186488986015, "train/time_per_step_avg": 0.021939503252506255, "train/epoch_time_elapsed": 198.84891088306904, "train/estimated_remaining_minutes": 0.3577687387665113}
-{"step": 1520, "epoch": 0.05123019885406134, "timestamp": 1787090223.1094587, "loss": 3.5856658935546877, "grad_norm": 1.703125, "learning_rate": 0.0003, "train/total_time_seconds": 32.62315646559, "train/time_per_step_avg": 0.021783830672502516, "train/epoch_time_elapsed": 199.81081285327673, "train/estimated_remaining_minutes": 0.35055584798550654}
-{"step": 1540, "epoch": 0.05190428041793057, "timestamp": 1787090224.036868, "loss": 3.592951202392578, "grad_norm": 1.4296875, "learning_rate": 0.0003, "train/total_time_seconds": 33.04965380206704, "train/time_per_step_avg": 0.021746470779180526, "train/epoch_time_elapsed": 200.7382232248783, "train/estimated_remaining_minutes": 0.34337302651498225}
-{"step": 1560, "epoch": 0.0525783619817998, "timestamp": 1787090224.9690807, "loss": 3.5940937042236327, "grad_norm": 1.8046875, "learning_rate": 0.0003, "train/total_time_seconds": 33.478857297450304, "train/time_per_step_avg": 0.021654499880969524, "train/epoch_time_elapsed": 201.67043430358171, "train/estimated_remaining_minutes": 0.33621929337183}
-{"step": 1580, "epoch": 0.05325244354566903, "timestamp": 1787090225.8977115, "loss": 3.5703506469726562, "grad_norm": 1.375, "learning_rate": 0.0003, "train/total_time_seconds": 33.90842756256461, "train/time_per_step_avg": 0.021575471162796022, "train/epoch_time_elapsed": 202.59906630590558, "train/estimated_remaining_minutes": 0.32906912824429796}
-{"step": 1600, "epoch": 0.053926525109538256, "timestamp": 1787090226.8264098, "loss": 3.5667308807373046, "grad_norm": 1.2578125, "learning_rate": 0.0003, "train/total_time_seconds": 34.33643752709031, "train/time_per_step_avg": 0.02137251038104296, "train/epoch_time_elapsed": 203.5277648679912, "train/estimated_remaining_minutes": 0.32190410181647167}
-{"step": 1600, "epoch": 0.053926525109538256, "timestamp": 1787090235.2771208, "eval_loss": 3.55812931060791, "eval_runtime": 8.4492, "eval_samples_per_second": 1127.557, "eval_steps_per_second": 0.828, "train/total_time_seconds": 34.33643752709031, "train/time_per_step_avg": 0.02137251038104296, "train/epoch_time_elapsed": 211.97847399488091, "train/estimated_remaining_minutes": 0.32190410181647167}
-{"step": 1620, "epoch": 0.054600606673407485, "timestamp": 1787090236.2274826, "loss": 3.578482437133789, "grad_norm": 1.3359375, "learning_rate": 0.0003, "train/total_time_seconds": 34.754911847412586, "train/time_per_step_avg": 0.02131755381822586, "train/epoch_time_elapsed": 212.92883710190654, "train/estimated_remaining_minutes": 0.31465352289838555}
-{"step": 1640, "epoch": 0.05527468823727671, "timestamp": 1787090237.15594, "loss": 3.5558090209960938, "grad_norm": 1.3984375, "learning_rate": 0.0003, "train/total_time_seconds": 35.1784917190671, "train/time_per_step_avg": 0.02128837917000055, "train/epoch_time_elapsed": 213.8572948165238, "train/estimated_remaining_minutes": 0.30745429754469206}
-{"step": 1660, "epoch": 0.05594876980114594, "timestamp": 1787090238.085788, "loss": 3.566594696044922, "grad_norm": 1.359375, "learning_rate": 0.0003, "train/total_time_seconds": 35.59984588995576, "train/time_per_step_avg": 0.02120988592505455, "train/epoch_time_elapsed": 214.78714298456907, "train/estimated_remaining_minutes": 0.30023966413215697}
-{"step": 1680, "epoch": 0.056622851365015166, "timestamp": 1787090239.0095909, "loss": 3.5346706390380858, "grad_norm": 1.6015625, "learning_rate": 0.0003, "train/total_time_seconds": 36.021644342690706, "train/time_per_step_avg": 0.021132167801260947, "train/epoch_time_elapsed": 215.71094546467066, "train/estimated_remaining_minutes": 0.2930332178671267}
-{"step": 1700, "epoch": 0.057296932928884395, "timestamp": 1787090239.9389343, "loss": 3.530739974975586, "grad_norm": 1.296875, "learning_rate": 0.0003, "train/total_time_seconds": 36.447514064610004, "train/time_per_step_avg": 0.021110765375196933, "train/epoch_time_elapsed": 216.64028869196773, "train/estimated_remaining_minutes": 0.2858628554087059}
-{"step": 1700, "epoch": 0.057296932928884395, "timestamp": 1787090248.842449, "eval_loss": 3.530102014541626, "eval_runtime": 8.9019, "eval_samples_per_second": 1070.223, "eval_steps_per_second": 0.786, "train/total_time_seconds": 36.447514064610004, "train/time_per_step_avg": 0.021110765375196933, "train/epoch_time_elapsed": 225.54380098730326, "train/estimated_remaining_minutes": 0.2858628554087059}
-{"step": 1720, "epoch": 0.057971014492753624, "timestamp": 1787090249.8342133, "loss": 3.519282913208008, "grad_norm": 1.4296875, "learning_rate": 0.0003, "train/total_time_seconds": 36.8907287530601, "train/time_per_step_avg": 0.02135816905647516, "train/epoch_time_elapsed": 226.53556845709682, "train/estimated_remaining_minutes": 0.2788252754591752}
-{"step": 1740, "epoch": 0.05864509605662285, "timestamp": 1787090250.8161082, "loss": 3.5207279205322264, "grad_norm": 1.5078125, "learning_rate": 0.0003, "train/total_time_seconds": 37.33156434074044, "train/time_per_step_avg": 0.021530726216733454, "train/epoch_time_elapsed": 227.51746324449778, "train/estimated_remaining_minutes": 0.2717623457755051}
-{"step": 1760, "epoch": 0.05931917762049208, "timestamp": 1787090251.812636, "loss": 3.5107025146484374, "grad_norm": 1.4140625, "learning_rate": 0.0003, "train/total_time_seconds": 37.78914465382695, "train/time_per_step_avg": 0.02189298763871193, "train/epoch_time_elapsed": 228.51399037614465, "train/estimated_remaining_minutes": 0.2648102939756813}
-{"step": 1780, "epoch": 0.05999325918436131, "timestamp": 1787090252.737064, "loss": 3.5151645660400392, "grad_norm": 1.4921875, "learning_rate": 0.0003, "train/total_time_seconds": 38.21554037556052, "train/time_per_step_avg": 0.02193896032869816, "train/epoch_time_elapsed": 229.43841843679547, "train/estimated_remaining_minutes": 0.25763285646445294}
-{"step": 1800, "epoch": 0.06066734074823053, "timestamp": 1787090253.6519675, "loss": 3.506744384765625, "grad_norm": 1.34375, "learning_rate": 0.0003, "train/total_time_seconds": 38.63394458219409, "train/time_per_step_avg": 0.021864305175840856, "train/epoch_time_elapsed": 230.35332256928086, "train/estimated_remaining_minutes": 0.2504051963660728}
-{"step": 1800, "epoch": 0.06066734074823053, "timestamp": 1787090262.13157, "eval_loss": 3.510505437850952, "eval_runtime": 8.4781, "eval_samples_per_second": 1123.716, "eval_steps_per_second": 0.826, "train/total_time_seconds": 38.63394458219409, "train/time_per_step_avg": 0.021864305175840856, "train/epoch_time_elapsed": 238.83292232453823, "train/estimated_remaining_minutes": 0.2504051963660728}
-{"step": 1820, "epoch": 0.06134142231209976, "timestamp": 1787090263.1293309, "loss": 3.4982723236083983, "grad_norm": 1.265625, "learning_rate": 0.0003, "train/total_time_seconds": 39.06876355037093, "train/time_per_step_avg": 0.02178034797310829, "train/epoch_time_elapsed": 239.83068535104394, "train/estimated_remaining_minutes": 0.24328534078985564}
-{"step": 1840, "epoch": 0.06201550387596899, "timestamp": 1787090264.0554924, "loss": 3.4931972503662108, "grad_norm": 1.6015625, "learning_rate": 0.0003, "train/total_time_seconds": 39.49600052088499, "train/time_per_step_avg": 0.021644361801445484, "train/epoch_time_elapsed": 240.75684678554535, "train/estimated_remaining_minutes": 0.23611739441833418}
-{"step": 1860, "epoch": 0.06268958543983821, "timestamp": 1787090264.9857557, "loss": 3.5061672210693358, "grad_norm": 1.296875, "learning_rate": 0.0003, "train/total_time_seconds": 39.91950794309378, "train/time_per_step_avg": 0.021303632892668248, "train/epoch_time_elapsed": 241.68710988014936, "train/estimated_remaining_minutes": 0.2289290778098568}
-{"step": 1880, "epoch": 0.06336366700370745, "timestamp": 1787090265.9066734, "loss": 3.4790119171142577, "grad_norm": 1.7421875, "learning_rate": 0.0003, "train/total_time_seconds": 40.34255151450634, "train/time_per_step_avg": 0.02127011138945818, "train/epoch_time_elapsed": 242.60802837461233, "train/estimated_remaining_minutes": 0.22174097463647102}
-{"step": 1900, "epoch": 0.06403774856757667, "timestamp": 1787090266.8239949, "loss": 3.4856170654296874, "grad_norm": 1.4296875, "learning_rate": 0.0003, "train/total_time_seconds": 40.76226783171296, "train/time_per_step_avg": 0.021283232495188712, "train/epoch_time_elapsed": 243.5253492332995, "train/estimated_remaining_minutes": 0.2145382517458577}
-{"step": 1900, "epoch": 0.06403774856757667, "timestamp": 1787090275.2962322, "eval_loss": 3.486959457397461, "eval_runtime": 8.4707, "eval_samples_per_second": 1124.702, "eval_steps_per_second": 0.826, "train/total_time_seconds": 40.76226783171296, "train/time_per_step_avg": 0.021283232495188712, "train/epoch_time_elapsed": 251.99758548289537, "train/estimated_remaining_minutes": 0.2145382517458577}
-{"step": 1920, "epoch": 0.06471183013144591, "timestamp": 1787090276.2503629, "loss": 3.4741825103759765, "grad_norm": 1.625, "learning_rate": 0.0003, "train/total_time_seconds": 41.185888312757015, "train/time_per_step_avg": 0.021171247623860835, "train/epoch_time_elapsed": 252.95171785727143, "train/estimated_remaining_minutes": 0.20735950713020024}
-{"step": 1940, "epoch": 0.06538591169531513, "timestamp": 1787090277.180064, "loss": 3.459843063354492, "grad_norm": 1.484375, "learning_rate": 0.0003, "train/total_time_seconds": 41.61065972223878, "train/time_per_step_avg": 0.021146592013537885, "train/epoch_time_elapsed": 253.8814189210534, "train/estimated_remaining_minutes": 0.20018874093173297}
-{"step": 1960, "epoch": 0.06605999325918437, "timestamp": 1787090278.1718454, "loss": 3.482832336425781, "grad_norm": 1.3359375, "learning_rate": 0.0003, "train/total_time_seconds": 42.0616734996438, "train/time_per_step_avg": 0.02142165556550026, "train/epoch_time_elapsed": 254.873200006783, "train/estimated_remaining_minutes": 0.1931403374983644}
-{"step": 1980, "epoch": 0.06673407482305359, "timestamp": 1787090279.1152725, "loss": 3.490843963623047, "grad_norm": 1.296875, "learning_rate": 0.0003, "train/total_time_seconds": 42.502584993839264, "train/time_per_step_avg": 0.021600334793329238, "train/epoch_time_elapsed": 255.81662721559405, "train/estimated_remaining_minutes": 0.18603825081478467}
-{"step": 2000, "epoch": 0.06740815638692282, "timestamp": 1787090280.0470285, "loss": 3.471944046020508, "grad_norm": 1.609375, "learning_rate": 0.0003, "train/total_time_seconds": 42.93809688463807, "train/time_per_step_avg": 0.021758290529251097, "train/epoch_time_elapsed": 256.7483834400773, "train/estimated_remaining_minutes": 0.1789087370193253}
-{"step": 2000, "epoch": 0.06740815638692282, "timestamp": 1787090288.5251565, "eval_loss": 3.4656903743743896, "eval_runtime": 8.4764, "eval_samples_per_second": 1123.949, "eval_steps_per_second": 0.826, "train/total_time_seconds": 42.93809688463807, "train/time_per_step_avg": 0.021758290529251097, "train/epoch_time_elapsed": 265.22650999203324, "train/estimated_remaining_minutes": 0.1789087370193253}
-{"step": 2020, "epoch": 0.06808223795079205, "timestamp": 1787090289.4874835, "loss": 3.4606929779052735, "grad_norm": 1.8046875, "learning_rate": 0.0003, "train/total_time_seconds": 43.36348307877779, "train/time_per_step_avg": 0.02177594766020775, "train/epoch_time_elapsed": 266.188837967813, "train/estimated_remaining_minutes": 0.1717365666486249}
-{"step": 2040, "epoch": 0.06875631951466127, "timestamp": 1787090290.4118824, "loss": 3.4594757080078127, "grad_norm": 1.515625, "learning_rate": 0.0003, "train/total_time_seconds": 43.79138084128499, "train/time_per_step_avg": 0.02180721119046211, "train/epoch_time_elapsed": 267.1132374033332, "train/estimated_remaining_minutes": 0.16457545087411027}
-{"step": 2060, "epoch": 0.0694304010785305, "timestamp": 1787090291.3374932, "loss": 3.446879577636719, "grad_norm": 1.2265625, "learning_rate": 0.0003, "train/total_time_seconds": 44.2190666384995, "train/time_per_step_avg": 0.02157393138855696, "train/epoch_time_elapsed": 268.0388476587832, "train/estimated_remaining_minutes": 0.15741415308203704}
-{"step": 2080, "epoch": 0.07010448264239973, "timestamp": 1787090292.3561628, "loss": 3.4532211303710936, "grad_norm": 1.34375, "learning_rate": 0.0003, "train/total_time_seconds": 44.67429492995143, "train/time_per_step_avg": 0.021717099361121654, "train/epoch_time_elapsed": 269.0575177781284, "train/estimated_remaining_minutes": 0.15034618486041346}
-{"step": 2100, "epoch": 0.07077856420626896, "timestamp": 1787090293.307748, "loss": 3.445978546142578, "grad_norm": 1.21875, "learning_rate": 0.0003, "train/total_time_seconds": 45.11734665185213, "train/time_per_step_avg": 0.021792497672140598, "train/epoch_time_elapsed": 270.0091031193733, "train/estimated_remaining_minutes": 0.14322967191064168}
-{"step": 2100, "epoch": 0.07077856420626896, "timestamp": 1787090301.7926538, "eval_loss": 3.4454257488250732, "eval_runtime": 8.4831, "eval_samples_per_second": 1123.053, "eval_steps_per_second": 0.825, "train/total_time_seconds": 45.11734665185213, "train/time_per_step_avg": 0.021792497672140598, "train/epoch_time_elapsed": 278.49400681629777, "train/estimated_remaining_minutes": 0.14322967191064168}
-{"step": 2120, "epoch": 0.07145264577013818, "timestamp": 1787090302.752118, "loss": 3.446368408203125, "grad_norm": 1.3984375, "learning_rate": 0.0003, "train/total_time_seconds": 45.54672980308533, "train/time_per_step_avg": 0.02183246724307537, "train/epoch_time_elapsed": 279.4534733183682, "train/estimated_remaining_minutes": 0.13606727456896558}
-{"step": 2140, "epoch": 0.07212672733400742, "timestamp": 1787090303.6751964, "loss": 3.4467777252197265, "grad_norm": 1.9453125, "learning_rate": 0.0003, "train/total_time_seconds": 45.97253290563822, "train/time_per_step_avg": 0.021811520643532277, "train/epoch_time_elapsed": 280.37655137851834, "train/estimated_remaining_minutes": 0.12889495207188284}
-{"step": 2160, "epoch": 0.07280080889787664, "timestamp": 1787090304.601299, "loss": 3.427969741821289, "grad_norm": 1.2421875, "learning_rate": 0.0003, "train/total_time_seconds": 46.40290119126439, "train/time_per_step_avg": 0.021838345527648927, "train/epoch_time_elapsed": 281.30265340954065, "train/estimated_remaining_minutes": 0.1217360062116504}
-{"step": 2180, "epoch": 0.07347489046174586, "timestamp": 1787090305.5422475, "loss": 3.437174606323242, "grad_norm": 1.546875, "learning_rate": 0.0003, "train/total_time_seconds": 46.82929648458958, "train/time_per_step_avg": 0.021550015546381474, "train/epoch_time_elapsed": 282.2436023950577, "train/estimated_remaining_minutes": 0.11456708620083077}
-{"step": 2200, "epoch": 0.0741489720256151, "timestamp": 1787090306.4651196, "loss": 3.437343215942383, "grad_norm": 1.5, "learning_rate": 0.0003, "train/total_time_seconds": 47.253708347678185, "train/time_per_step_avg": 0.021363616958260535, "train/epoch_time_elapsed": 283.16647424921393, "train/estimated_remaining_minutes": 0.1073947916992686}
-{"step": 2200, "epoch": 0.0741489720256151, "timestamp": 1787090314.8796115, "eval_loss": 3.431370735168457, "eval_runtime": 8.4127, "eval_samples_per_second": 1132.457, "eval_steps_per_second": 0.832, "train/total_time_seconds": 47.253708347678185, "train/time_per_step_avg": 0.021363616958260535, "train/epoch_time_elapsed": 291.5809638015926, "train/estimated_remaining_minutes": 0.1073947916992686}
-{"step": 2220, "epoch": 0.07482305358948432, "timestamp": 1787090315.8496833, "loss": 3.3987106323242187, "grad_norm": 1.265625, "learning_rate": 0.0003, "train/total_time_seconds": 47.68398727849126, "train/time_per_step_avg": 0.021372574754059313, "train/epoch_time_elapsed": 292.55103803798556, "train/estimated_remaining_minutes": 0.10023660989472637}
-{"step": 2240, "epoch": 0.07549713515335356, "timestamp": 1787090316.7768753, "loss": 3.4200679779052736, "grad_norm": 1.25, "learning_rate": 0.0003, "train/total_time_seconds": 48.1130493208766, "train/time_per_step_avg": 0.021405164152383804, "train/epoch_time_elapsed": 293.47823068127036, "train/estimated_remaining_minutes": 0.09307583946002912}
-{"step": 2260, "epoch": 0.07617121671722278, "timestamp": 1787090317.712523, "loss": 3.391765594482422, "grad_norm": 1.46875, "learning_rate": 0.0003, "train/total_time_seconds": 48.5431542173028, "train/time_per_step_avg": 0.021402530260384082, "train/epoch_time_elapsed": 294.4138778038323, "train/estimated_remaining_minutes": 0.08591708711027043}
-{"step": 2280, "epoch": 0.07684529828109202, "timestamp": 1787090318.6368556, "loss": 3.4147933959960937, "grad_norm": 1.546875, "learning_rate": 0.0003, "train/total_time_seconds": 48.96943225711584, "train/time_per_step_avg": 0.02140135772526264, "train/epoch_time_elapsed": 295.3382097110152, "train/estimated_remaining_minutes": 0.07875201093980616}
-{"step": 2300, "epoch": 0.07751937984496124, "timestamp": 1787090319.5716066, "loss": 3.411570358276367, "grad_norm": 1.4609375, "learning_rate": 0.0003, "train/total_time_seconds": 49.40087690204382, "train/time_per_step_avg": 0.02147168554365635, "train/epoch_time_elapsed": 296.272961769253, "train/estimated_remaining_minutes": 0.071595473771078}
-{"step": 2300, "epoch": 0.07751937984496124, "timestamp": 1787090327.969581, "eval_loss": 3.4143009185791016, "eval_runtime": 8.3964, "eval_samples_per_second": 1134.65, "eval_steps_per_second": 0.834, "train/total_time_seconds": 49.40087690204382, "train/time_per_step_avg": 0.02147168554365635, "train/epoch_time_elapsed": 304.6709332689643, "train/estimated_remaining_minutes": 0.071595473771078}
-{"step": 2320, "epoch": 0.07819346140883048, "timestamp": 1787090328.9620771, "loss": 3.408540725708008, "grad_norm": 1.4765625, "learning_rate": 0.0003, "train/total_time_seconds": 49.833336658775806, "train/time_per_step_avg": 0.021493493802845477, "train/epoch_time_elapsed": 305.66343190521, "train/estimated_remaining_minutes": 0.06443965947255492}
-{"step": 2340, "epoch": 0.0788675429726997, "timestamp": 1787090329.8789449, "loss": 3.393010711669922, "grad_norm": 1.46875, "learning_rate": 0.0003, "train/total_time_seconds": 50.2537716627121, "train/time_per_step_avg": 0.02140722341835499, "train/epoch_time_elapsed": 306.58029908686876, "train/estimated_remaining_minutes": 0.05726925545608216}
-{"step": 2360, "epoch": 0.07954162453656892, "timestamp": 1787090330.7985592, "loss": 3.4179660797119142, "grad_norm": 1.3359375, "learning_rate": 0.0003, "train/total_time_seconds": 50.677648298442364, "train/time_per_step_avg": 0.021344940811395645, "train/epoch_time_elapsed": 307.4999114535749, "train/estimated_remaining_minutes": 0.050105019504109685}
-{"step": 2380, "epoch": 0.08021570610043816, "timestamp": 1787090331.7210913, "loss": 3.4083057403564454, "grad_norm": 1.5078125, "learning_rate": 0.0003, "train/total_time_seconds": 51.10111549496651, "train/time_per_step_avg": 0.02131683237850666, "train/epoch_time_elapsed": 308.42244643718004, "train/estimated_remaining_minutes": 0.04294211386131639}
-{"step": 2400, "epoch": 0.08088978766430738, "timestamp": 1787090332.637041, "loss": 3.405569839477539, "grad_norm": 1.6640625, "learning_rate": 0.0003, "train/total_time_seconds": 51.520632427185774, "train/time_per_step_avg": 0.021197555251419545, "train/epoch_time_elapsed": 309.3383958786726, "train/estimated_remaining_minutes": 0.03577821696332346}
-{"step": 2400, "epoch": 0.08088978766430738, "timestamp": 1787090341.087288, "eval_loss": 3.405479907989502, "eval_runtime": 8.4486, "eval_samples_per_second": 1127.639, "eval_steps_per_second": 0.829, "train/total_time_seconds": 51.520632427185774, "train/time_per_step_avg": 0.021197555251419545, "train/epoch_time_elapsed": 317.7886408865452, "train/estimated_remaining_minutes": 0.03577821696332346}
-{"step": 2420, "epoch": 0.08156386922817661, "timestamp": 1787090342.070789, "loss": 3.422397232055664, "grad_norm": 1.359375, "learning_rate": 0.0003, "train/total_time_seconds": 51.95656028389931, "train/time_per_step_avg": 0.021232236251235007, "train/epoch_time_elapsed": 318.77214378118515, "train/estimated_remaining_minutes": 0.028626204013167664}
-{"step": 2440, "epoch": 0.08223795079204584, "timestamp": 1787090343.04529, "loss": 3.372893524169922, "grad_norm": 1.5234375, "learning_rate": 0.0003, "train/total_time_seconds": 52.38708958029747, "train/time_per_step_avg": 0.021333179175853728, "train/epoch_time_elapsed": 319.74664357304573, "train/estimated_remaining_minutes": 0.021470118680449783}
-{"step": 2460, "epoch": 0.08291203235591507, "timestamp": 1787090344.0064027, "loss": 3.4056819915771483, "grad_norm": 1.5078125, "learning_rate": 0.0003, "train/total_time_seconds": 52.83087908104062, "train/time_per_step_avg": 0.02153230782598257, "train/epoch_time_elapsed": 320.7077578008175, "train/estimated_remaining_minutes": 0.01431731140407605}
-{"step": 2480, "epoch": 0.0835861139197843, "timestamp": 1787090344.98779, "loss": 3.3840850830078124, "grad_norm": 1.6328125, "learning_rate": 0.0003, "train/total_time_seconds": 53.28470703586936, "train/time_per_step_avg": 0.02183591540902853, "train/epoch_time_elapsed": 321.68914468213916, "train/estimated_remaining_minutes": 0.007161922988692117}
-{"step": 2500, "epoch": 0.08426019548365352, "timestamp": 1787090345.914262, "loss": 3.3768421173095704, "grad_norm": 1.3359375, "learning_rate": 0.0003, "train/total_time_seconds": 53.71119434013963, "train/time_per_step_avg": 0.021905619129538537, "train/epoch_time_elapsed": 322.6156167127192, "train/estimated_remaining_minutes": 0.0}
-{"step": 2500, "epoch": 0.08426019548365352, "timestamp": 1787090354.314014, "eval_loss": 3.386671304702759, "eval_runtime": 8.3982, "eval_samples_per_second": 1134.413, "eval_steps_per_second": 0.834, "train/total_time_seconds": 53.71119434013963, "train/time_per_step_avg": 0.021905619129538537, "train/epoch_time_elapsed": 331.015366576612, "train/estimated_remaining_minutes": 0.0}
-{"step": 2500, "epoch": 0.08426019548365352, "timestamp": 1787090354.3694549, "train_runtime": 332.0558, "train_samples_per_second": 240.923, "train_steps_per_second": 7.529, "total_flos": 362985553920000.0, "train_loss": 4.1883163818359375, "train/total_time_seconds": 53.71119434013963, "train/time_per_step_avg": 0.021905619129538537, "train/epoch_time_elapsed": 331.0708075389266, "train/estimated_remaining_minutes": 0.0}
-{"step": 2500, "epoch": 0.08426019548365352, "timestamp": 1787090363.1779864, "eval_loss": 3.386671304702759, "eval_runtime": 8.8055, "eval_samples_per_second": 1081.938, "eval_steps_per_second": 0.795, "train/total_time_seconds": 53.71119434013963, "train/time_per_step_avg": 0.021905619129538537, "train/epoch_time_elapsed": 339.8793397396803, "train/estimated_remaining_minutes": 0.0}
+{"step": 20, "epoch": 0.0016852039096730705, "timestamp": 1787159744.744951, "loss": 8.200721740722656, "grad_norm": 1.2421875, "learning_rate": 9.5e-05, "train/total_time_seconds": 1.1820026859641075, "train/time_per_step_avg": 0.059100134298205376, "train/epoch_time_elapsed": 2.652187306433916, "train/estimated_remaining_minutes": 0.9653021935373545}
+{"step": 40, "epoch": 0.003370407819346141, "timestamp": 1787159746.6767337, "loss": 7.764720153808594, "grad_norm": 1.2421875, "learning_rate": 0.00019500000000000002, "train/total_time_seconds": 1.8186652921140194, "train/time_per_step_avg": 0.04546663230285049, "train/epoch_time_elapsed": 4.583965107798576, "train/estimated_remaining_minutes": 0.7274661168456078}
+{"step": 60, "epoch": 0.005055611729019211, "timestamp": 1787159748.5245914, "loss": 7.160160064697266, "grad_norm": 1.2265625, "learning_rate": 0.000295, "train/total_time_seconds": 2.4542956836521626, "train/time_per_step_avg": 0.04090492806086938, "train/epoch_time_elapsed": 6.4318275190889835, "train/estimated_remaining_minutes": 0.6408438729536203}
+{"step": 80, "epoch": 0.006740815638692282, "timestamp": 1787159750.3386374, "loss": 6.480591583251953, "grad_norm": 1.2109375, "learning_rate": 0.000395, "train/total_time_seconds": 3.086422026157379, "train/time_per_step_avg": 0.03858027532696724, "train/epoch_time_elapsed": 8.245873432606459, "train/estimated_remaining_minutes": 0.5915642216801643}
+{"step": 100, "epoch": 0.008426019548365353, "timestamp": 1787159752.2435765, "loss": 5.919136428833008, "grad_norm": 1.0078125, "learning_rate": 0.000495, "train/total_time_seconds": 3.722423255443573, "train/time_per_step_avg": 0.03722423255443573, "train/epoch_time_elapsed": 10.150811605155468, "train/estimated_remaining_minutes": 0.5583634883165359}
+{"step": 100, "epoch": 0.008426019548365353, "timestamp": 1787159760.215479, "eval_loss": 5.634258270263672, "eval_runtime": 7.97, "eval_samples_per_second": 1195.356, "eval_steps_per_second": 0.878, "train/total_time_seconds": 3.722423255443573, "train/time_per_step_avg": 0.03722423255443573, "train/epoch_time_elapsed": 18.122713901102543, "train/estimated_remaining_minutes": 0.5583634883165359}
+{"step": 120, "epoch": 0.010111223458038422, "timestamp": 1787159762.0986185, "loss": 5.412916946411133, "grad_norm": 1.171875, "learning_rate": 0.0005949999999999999, "train/total_time_seconds": 4.363878160715103, "train/time_per_step_avg": 0.031818754747509954, "train/epoch_time_elapsed": 20.005853559821844, "train/estimated_remaining_minutes": 0.5333628863096237}
+{"step": 140, "epoch": 0.011796427367711493, "timestamp": 1787159763.9296083, "loss": 5.0327880859375, "grad_norm": 1.515625, "learning_rate": 0.000695, "train/total_time_seconds": 5.001869980245829, "train/time_per_step_avg": 0.03183204688131809, "train/epoch_time_elapsed": 21.836841892451048, "train/estimated_remaining_minutes": 0.5120962122632634}
+{"step": 160, "epoch": 0.013481631277384564, "timestamp": 1787159765.7692258, "loss": 4.69476089477539, "grad_norm": 0.9765625, "learning_rate": 0.000795, "train/total_time_seconds": 5.6416174322366714, "train/time_per_step_avg": 0.03187321748584509, "train/epoch_time_elapsed": 23.676462575793266, "train/estimated_remaining_minutes": 0.4936415253207088}
+{"step": 180, "epoch": 0.015166835187057633, "timestamp": 1787159767.6319602, "loss": 4.4246673583984375, "grad_norm": 0.8828125, "learning_rate": 0.0008950000000000001, "train/total_time_seconds": 6.286054410040379, "train/time_per_step_avg": 0.03199632383882999, "train/epoch_time_elapsed": 25.539197202771902, "train/estimated_remaining_minutes": 0.4772745015030658}
+{"step": 200, "epoch": 0.016852039096730706, "timestamp": 1787159769.4737778, "loss": 4.204695129394532, "grad_norm": 0.87890625, "learning_rate": 0.000995, "train/total_time_seconds": 6.9344093687832355, "train/time_per_step_avg": 0.03211986113339663, "train/epoch_time_elapsed": 27.38101428002119, "train/estimated_remaining_minutes": 0.4622939579188824}
+{"step": 200, "epoch": 0.016852039096730706, "timestamp": 1787159777.4086134, "eval_loss": 4.133424282073975, "eval_runtime": 7.9329, "eval_samples_per_second": 1200.942, "eval_steps_per_second": 0.882, "train/total_time_seconds": 6.9344093687832355, "train/time_per_step_avg": 0.03211986113339663, "train/epoch_time_elapsed": 35.31584985554218, "train/estimated_remaining_minutes": 0.4622939579188824}
+{"step": 220, "epoch": 0.018537243006403775, "timestamp": 1787159779.2836711, "loss": 4.048733520507812, "grad_norm": 0.6875, "learning_rate": 0.001, "train/total_time_seconds": 7.581706669181585, "train/time_per_step_avg": 0.032178285084664825, "train/epoch_time_elapsed": 37.19090821221471, "train/estimated_remaining_minutes": 0.4480099395425482}
+{"step": 240, "epoch": 0.020222446916076844, "timestamp": 1787159781.146838, "loss": 3.9190834045410154, "grad_norm": 0.73828125, "learning_rate": 0.001, "train/total_time_seconds": 8.228051170706749, "train/time_per_step_avg": 0.0322618119046092, "train/epoch_time_elapsed": 39.054074600338936, "train/estimated_remaining_minutes": 0.434258256231745}
+{"step": 260, "epoch": 0.021907650825749917, "timestamp": 1787159782.9797618, "loss": 3.7833847045898437, "grad_norm": 0.68359375, "learning_rate": 0.001, "train/total_time_seconds": 8.87311216443777, "train/time_per_step_avg": 0.03231494732201099, "train/epoch_time_elapsed": 40.886998776346445, "train/estimated_remaining_minutes": 0.420904038569484}
+{"step": 280, "epoch": 0.023592854735422986, "timestamp": 1787159784.913182, "loss": 3.700960159301758, "grad_norm": 0.82421875, "learning_rate": 0.001, "train/total_time_seconds": 9.516451723873615, "train/time_per_step_avg": 0.032303973138332366, "train/epoch_time_elapsed": 42.82041900604963, "train/estimated_remaining_minutes": 0.4078479310231549}
+{"step": 300, "epoch": 0.025278058645096056, "timestamp": 1787159786.7454824, "loss": 3.6359439849853517, "grad_norm": 0.984375, "learning_rate": 0.001, "train/total_time_seconds": 10.157686203718185, "train/time_per_step_avg": 0.032232768349349496, "train/epoch_time_elapsed": 44.65271816775203, "train/estimated_remaining_minutes": 0.3950211301445961}
+{"step": 300, "epoch": 0.025278058645096056, "timestamp": 1787159794.7201555, "eval_loss": 3.5975606441497803, "eval_runtime": 7.973, "eval_samples_per_second": 1194.91, "eval_steps_per_second": 0.878, "train/total_time_seconds": 10.157686203718185, "train/time_per_step_avg": 0.032232768349349496, "train/epoch_time_elapsed": 52.627391546964645, "train/estimated_remaining_minutes": 0.3950211301445961}
+{"step": 320, "epoch": 0.026963262554769128, "timestamp": 1787159796.601689, "loss": 3.5393722534179686, "grad_norm": 0.80859375, "learning_rate": 0.001, "train/total_time_seconds": 10.796786732971668, "train/time_per_step_avg": 0.03215080063790083, "train/epoch_time_elapsed": 54.50892523676157, "train/estimated_remaining_minutes": 0.3823861967927466}
+{"step": 340, "epoch": 0.028648466464442197, "timestamp": 1787159798.453831, "loss": 3.492219924926758, "grad_norm": 0.72265625, "learning_rate": 0.001, "train/total_time_seconds": 11.440050598233938, "train/time_per_step_avg": 0.032119994275271894, "train/epoch_time_elapsed": 56.36106700450182, "train/estimated_remaining_minutes": 0.3701192840605098}
+{"step": 360, "epoch": 0.030333670374115267, "timestamp": 1787159800.2839177, "loss": 3.4430694580078125, "grad_norm": 0.8203125, "learning_rate": 0.001, "train/total_time_seconds": 12.078172758221626, "train/time_per_step_avg": 0.03205060593783855, "train/epoch_time_elapsed": 58.19115437567234, "train/estimated_remaining_minutes": 0.35787178542878895}
+{"step": 380, "epoch": 0.032018874283788336, "timestamp": 1787159802.1708894, "loss": 3.3989883422851563, "grad_norm": 0.8203125, "learning_rate": 0.001, "train/total_time_seconds": 12.717796016484499, "train/time_per_step_avg": 0.03201344292610884, "train/epoch_time_elapsed": 60.07812675833702, "train/estimated_remaining_minutes": 0.34583480395703464}
+{"step": 400, "epoch": 0.03370407819346141, "timestamp": 1787159804.0363617, "loss": 3.341664123535156, "grad_norm": 0.6484375, "learning_rate": 0.001, "train/total_time_seconds": 13.358520355075598, "train/time_per_step_avg": 0.03200834151357412, "train/epoch_time_elapsed": 61.94359891861677, "train/estimated_remaining_minutes": 0.33396300887688996}
+{"step": 400, "epoch": 0.03370407819346141, "timestamp": 1787159812.2065763, "eval_loss": 3.3295202255249023, "eval_runtime": 8.1687, "eval_samples_per_second": 1166.287, "eval_steps_per_second": 0.857, "train/total_time_seconds": 13.358520355075598, "train/time_per_step_avg": 0.03200834151357412, "train/epoch_time_elapsed": 70.11381255835295, "train/estimated_remaining_minutes": 0.33396300887688996}
+{"step": 420, "epoch": 0.03538928210313448, "timestamp": 1787159814.10439, "loss": 3.3047794342041015, "grad_norm": 0.7109375, "learning_rate": 0.001, "train/total_time_seconds": 14.000833839178085, "train/time_per_step_avg": 0.032040471062064174, "train/epoch_time_elapsed": 72.01162526011467, "train/estimated_remaining_minutes": 0.32224141375886073}
+{"step": 440, "epoch": 0.03707448601280755, "timestamp": 1787159816.047692, "loss": 3.258700942993164, "grad_norm": 0.71875, "learning_rate": 0.001, "train/total_time_seconds": 14.651910278946161, "train/time_per_step_avg": 0.03211859680712223, "train/epoch_time_elapsed": 73.95492927730083, "train/estimated_remaining_minutes": 0.31079809682613063}
+{"step": 460, "epoch": 0.03875968992248062, "timestamp": 1787159817.9195716, "loss": 3.229313278198242, "grad_norm": 0.73046875, "learning_rate": 0.001, "train/total_time_seconds": 15.297841779887676, "train/time_per_step_avg": 0.0321966902166605, "train/epoch_time_elapsed": 75.82680872827768, "train/estimated_remaining_minutes": 0.29930560004128065}
+{"step": 480, "epoch": 0.04044489383215369, "timestamp": 1787159819.7689188, "loss": 3.202067565917969, "grad_norm": 0.703125, "learning_rate": 0.001, "train/total_time_seconds": 15.936651539057493, "train/time_per_step_avg": 0.03218855522572994, "train/epoch_time_elapsed": 77.67615579813719, "train/estimated_remaining_minutes": 0.2877450972329825}
+{"step": 500, "epoch": 0.04213009774182676, "timestamp": 1787159821.6968458, "loss": 3.163772201538086, "grad_norm": 0.7265625, "learning_rate": 0.001, "train/total_time_seconds": 16.570955902338028, "train/time_per_step_avg": 0.0321243554726243, "train/epoch_time_elapsed": 79.60408221185207, "train/estimated_remaining_minutes": 0.27618259837230047}
+{"step": 500, "epoch": 0.04213009774182676, "timestamp": 1787159829.648704, "eval_loss": 3.157355308532715, "eval_runtime": 7.9503, "eval_samples_per_second": 1198.315, "eval_steps_per_second": 0.88, "train/total_time_seconds": 16.570955902338028, "train/time_per_step_avg": 0.0321243554726243, "train/epoch_time_elapsed": 87.5559413023293, "train/estimated_remaining_minutes": 0.27618259837230047}
+{"step": 520, "epoch": 0.043815301651499834, "timestamp": 1787159831.4997501, "loss": 3.1448848724365233, "grad_norm": 0.6953125, "learning_rate": 0.001, "train/total_time_seconds": 17.204289257526398, "train/time_per_step_avg": 0.032034554183483124, "train/epoch_time_elapsed": 89.40698843076825, "train/estimated_remaining_minutes": 0.2646813731927138}
+{"step": 540, "epoch": 0.0455005055611729, "timestamp": 1787159833.3223414, "loss": 3.103527069091797, "grad_norm": 0.828125, "learning_rate": 0.001, "train/total_time_seconds": 17.837791569530964, "train/time_per_step_avg": 0.03185881290584803, "train/epoch_time_elapsed": 91.22957941517234, "train/estimated_remaining_minutes": 0.2532525963575384}
+{"step": 560, "epoch": 0.04718570947084597, "timestamp": 1787159835.1489348, "loss": 3.08404541015625, "grad_norm": 0.7578125, "learning_rate": 0.001, "train/total_time_seconds": 18.47138799726963, "train/time_per_step_avg": 0.03173546217381954, "train/epoch_time_elapsed": 93.05617316067219, "train/estimated_remaining_minutes": 0.241887223773769}
+{"step": 580, "epoch": 0.04887091338051904, "timestamp": 1787159836.9735324, "loss": 3.0501741409301757, "grad_norm": 0.76953125, "learning_rate": 0.001, "train/total_time_seconds": 19.10541071370244, "train/time_per_step_avg": 0.03168759174644947, "train/epoch_time_elapsed": 94.88077012076974, "train/estimated_remaining_minutes": 0.23058254309640874}
+{"step": 600, "epoch": 0.05055611729019211, "timestamp": 1787159839.033019, "loss": 3.0376760482788088, "grad_norm": 0.96875, "learning_rate": 0.001, "train/total_time_seconds": 19.750634588301182, "train/time_per_step_avg": 0.03179678685963154, "train/epoch_time_elapsed": 96.94025699794292, "train/estimated_remaining_minutes": 0.2194514954255687}
+{"step": 600, "epoch": 0.05055611729019211, "timestamp": 1787159847.154113, "eval_loss": 3.0310890674591064, "eval_runtime": 8.1195, "eval_samples_per_second": 1173.346, "eval_steps_per_second": 0.862, "train/total_time_seconds": 19.750634588301182, "train/time_per_step_avg": 0.03179678685963154, "train/epoch_time_elapsed": 105.06134972721338, "train/estimated_remaining_minutes": 0.2194514954255687}
+{"step": 620, "epoch": 0.05224132119986518, "timestamp": 1787159849.0143704, "loss": 3.0314205169677733, "grad_norm": 0.82421875, "learning_rate": 0.001, "train/total_time_seconds": 20.382843017578125, "train/time_per_step_avg": 0.03178553760051727, "train/epoch_time_elapsed": 106.92160803079605, "train/estimated_remaining_minutes": 0.2082118372763357}
+{"step": 640, "epoch": 0.053926525109538256, "timestamp": 1787159850.828231, "loss": 3.0014928817749023, "grad_norm": 0.640625, "learning_rate": 0.001, "train/total_time_seconds": 21.01229700446129, "train/time_per_step_avg": 0.03174505434930325, "train/epoch_time_elapsed": 108.73546917364001, "train/estimated_remaining_minutes": 0.19699028441682456}
+{"step": 660, "epoch": 0.055611729019211326, "timestamp": 1787159852.7167454, "loss": 2.9963293075561523, "grad_norm": 0.70703125, "learning_rate": 0.001, "train/total_time_seconds": 21.644355565309525, "train/time_per_step_avg": 0.031729675680398944, "train/epoch_time_elapsed": 110.62398328632116, "train/estimated_remaining_minutes": 0.18583537606578884}
+{"step": 680, "epoch": 0.057296932928884395, "timestamp": 1787159854.5207703, "loss": 2.9568761825561523, "grad_norm": 0.7109375, "learning_rate": 0.001, "train/total_time_seconds": 22.27377400547266, "train/time_per_step_avg": 0.031683632917702195, "train/epoch_time_elapsed": 112.42800731211901, "train/estimated_remaining_minutes": 0.1746962667095895}
+{"step": 700, "epoch": 0.058982136838557464, "timestamp": 1787159856.334935, "loss": 2.9395275115966797, "grad_norm": 0.69921875, "learning_rate": 0.001, "train/total_time_seconds": 22.903755206614733, "train/time_per_step_avg": 0.03153120618313551, "train/epoch_time_elapsed": 114.24217312037945, "train/estimated_remaining_minutes": 0.16359825147581952}
+{"step": 700, "epoch": 0.058982136838557464, "timestamp": 1787159864.3117049, "eval_loss": 2.939429759979248, "eval_runtime": 7.975, "eval_samples_per_second": 1194.602, "eval_steps_per_second": 0.878, "train/total_time_seconds": 22.903755206614733, "train/time_per_step_avg": 0.03153120618313551, "train/epoch_time_elapsed": 122.21894185245037, "train/estimated_remaining_minutes": 0.16359825147581952}
+{"step": 720, "epoch": 0.06066734074823053, "timestamp": 1787159866.154933, "loss": 2.922671890258789, "grad_norm": 0.6484375, "learning_rate": 0.001, "train/total_time_seconds": 23.537149403244257, "train/time_per_step_avg": 0.03154306385666132, "train/epoch_time_elapsed": 124.06217032298446, "train/estimated_remaining_minutes": 0.15255559798399052}
+{"step": 740, "epoch": 0.06235254465790361, "timestamp": 1787159867.9758906, "loss": 2.9145633697509767, "grad_norm": 0.95703125, "learning_rate": 0.001, "train/total_time_seconds": 24.17463242635131, "train/time_per_step_avg": 0.0316233542189002, "train/epoch_time_elapsed": 125.88312843069434, "train/estimated_remaining_minutes": 0.14156316285701215}
+{"step": 760, "epoch": 0.06403774856757667, "timestamp": 1787159869.9564908, "loss": 2.8876668930053713, "grad_norm": 0.7109375, "learning_rate": 0.001, "train/total_time_seconds": 24.810363072901964, "train/time_per_step_avg": 0.031660075075924395, "train/epoch_time_elapsed": 127.86372835934162, "train/estimated_remaining_minutes": 0.13058085827843138}
+{"step": 780, "epoch": 0.06572295247724974, "timestamp": 1787159871.839654, "loss": 2.8692466735839846, "grad_norm": 0.69140625, "learning_rate": 0.001, "train/total_time_seconds": 25.449912142008543, "train/time_per_step_avg": 0.03176138136535883, "train/epoch_time_elapsed": 129.74689135327935, "train/estimated_remaining_minutes": 0.11963633912909998}
+{"step": 800, "epoch": 0.06740815638692282, "timestamp": 1787159873.668295, "loss": 2.8788633346557617, "grad_norm": 0.68359375, "learning_rate": 0.001, "train/total_time_seconds": 26.088915783911943, "train/time_per_step_avg": 0.03185160577297211, "train/epoch_time_elapsed": 131.57553120702505, "train/estimated_remaining_minutes": 0.10870381576629977}
+{"step": 800, "epoch": 0.06740815638692282, "timestamp": 1787159881.9473228, "eval_loss": 2.8632190227508545, "eval_runtime": 8.2773, "eval_samples_per_second": 1150.973, "eval_steps_per_second": 0.846, "train/total_time_seconds": 26.088915783911943, "train/time_per_step_avg": 0.03185160577297211, "train/epoch_time_elapsed": 139.8545593805611, "train/estimated_remaining_minutes": 0.10870381576629977}
+{"step": 820, "epoch": 0.0690933602965959, "timestamp": 1787159883.8665261, "loss": 2.8496471405029298, "grad_norm": 0.6796875, "learning_rate": 0.001, "train/total_time_seconds": 26.722276385873556, "train/time_per_step_avg": 0.031851269826292994, "train/epoch_time_elapsed": 141.77376406639814, "train/estimated_remaining_minutes": 0.09776442580197643}
+{"step": 840, "epoch": 0.07077856420626896, "timestamp": 1787159885.6946013, "loss": 2.847636604309082, "grad_norm": 0.62109375, "learning_rate": 0.001, "train/total_time_seconds": 27.35644706711173, "train/time_per_step_avg": 0.03181814640760422, "train/epoch_time_elapsed": 143.60183906927705, "train/estimated_remaining_minutes": 0.0868458637051166}
+{"step": 860, "epoch": 0.07246376811594203, "timestamp": 1787159887.564113, "loss": 2.8339916229248048, "grad_norm": 0.73046875, "learning_rate": 0.001, "train/total_time_seconds": 27.989167381078005, "train/time_per_step_avg": 0.031788043081760406, "train/epoch_time_elapsed": 145.47135097533464, "train/estimated_remaining_minutes": 0.07593960142152947}
+{"step": 880, "epoch": 0.0741489720256151, "timestamp": 1787159889.387239, "loss": 2.8253028869628904, "grad_norm": 0.69921875, "learning_rate": 0.001, "train/total_time_seconds": 28.62036318704486, "train/time_per_step_avg": 0.03170451045036316, "train/epoch_time_elapsed": 147.29447646439075, "train/estimated_remaining_minutes": 0.0650462799705565}
+{"step": 900, "epoch": 0.07583417593528817, "timestamp": 1787159891.2191205, "loss": 2.7853120803833007, "grad_norm": 0.6875, "learning_rate": 0.001, "train/total_time_seconds": 29.254599027335644, "train/time_per_step_avg": 0.031656832434237, "train/epoch_time_elapsed": 149.1263581365347, "train/estimated_remaining_minutes": 0.0541751833839549}
+{"step": 900, "epoch": 0.07583417593528817, "timestamp": 1787159899.179946, "eval_loss": 2.8034849166870117, "eval_runtime": 7.9593, "eval_samples_per_second": 1196.967, "eval_steps_per_second": 0.879, "train/total_time_seconds": 29.254599027335644, "train/time_per_step_avg": 0.031656832434237, "train/epoch_time_elapsed": 157.0871831253171, "train/estimated_remaining_minutes": 0.0541751833839549}
+{"step": 920, "epoch": 0.07751937984496124, "timestamp": 1787159901.2376366, "loss": 2.794095993041992, "grad_norm": 0.609375, "learning_rate": 0.001, "train/total_time_seconds": 29.887034360319376, "train/time_per_step_avg": 0.0316475797444582, "train/epoch_time_elapsed": 159.14487295597792, "train/estimated_remaining_minutes": 0.0433145425511875}
+{"step": 940, "epoch": 0.07920458375463431, "timestamp": 1787159903.067668, "loss": 2.7890634536743164, "grad_norm": 0.671875, "learning_rate": 0.001, "train/total_time_seconds": 30.5212767906487, "train/time_per_step_avg": 0.03164829723536968, "train/epoch_time_elapsed": 160.9749058149755, "train/estimated_remaining_minutes": 0.03246944339430713}
+{"step": 960, "epoch": 0.08088978766430738, "timestamp": 1787159904.9205842, "loss": 2.784823989868164, "grad_norm": 0.58203125, "learning_rate": 0.001, "train/total_time_seconds": 31.156463339924812, "train/time_per_step_avg": 0.03167295958846807, "train/epoch_time_elapsed": 162.82782202214003, "train/estimated_remaining_minutes": 0.021636432874947785}
+{"step": 980, "epoch": 0.08257499157398045, "timestamp": 1787159906.7382162, "loss": 2.775278663635254, "grad_norm": 0.60546875, "learning_rate": 0.001, "train/total_time_seconds": 31.790426205843687, "train/time_per_step_avg": 0.03170063018798828, "train/epoch_time_elapsed": 164.64545308053493, "train/estimated_remaining_minutes": 0.010813070138042068}
+{"step": 1000, "epoch": 0.08426019548365352, "timestamp": 1787159908.6763556, "loss": 2.753749656677246, "grad_norm": 0.78125, "learning_rate": 0.001, "train/total_time_seconds": 32.43894848972559, "train/time_per_step_avg": 0.03184349462389946, "train/epoch_time_elapsed": 166.58359253406525, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.08426019548365352, "timestamp": 1787159916.6100245, "eval_loss": 2.7597591876983643, "eval_runtime": 7.9273, "eval_samples_per_second": 1201.797, "eval_steps_per_second": 0.883, "train/total_time_seconds": 32.43894848972559, "train/time_per_step_avg": 0.03184349462389946, "train/epoch_time_elapsed": 174.5172589495778, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.08426019548365352, "timestamp": 1787159916.6630983, "train_runtime": 175.4068, "train_samples_per_second": 456.083, "train_steps_per_second": 5.701, "total_flos": 362985553920000.0, "train_loss": 3.6923015975952147, "train/total_time_seconds": 32.43894848972559, "train/time_per_step_avg": 0.03184349462389946, "train/epoch_time_elapsed": 174.57033431902528, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.08426019548365352, "timestamp": 1787159924.561638, "eval_loss": 2.7597591876983643, "eval_runtime": 7.8957, "eval_samples_per_second": 1206.607, "eval_steps_per_second": 0.887, "train/total_time_seconds": 32.43894848972559, "train/time_per_step_avg": 0.03184349462389946, "train/epoch_time_elapsed": 182.46887450292706, "train/estimated_remaining_minutes": 0.0}
diff --git a/zain/Activation/out/sweep_summary.json b/zain/Activation/out/sweep_summary.json
index 87326af44e793c4d8133cebb212f97c6de4486fa..7e2188ec9dd77eebaafac8a9c5b58e75496f3af2 100644
--- a/zain/Activation/out/sweep_summary.json
+++ b/zain/Activation/out/sweep_summary.json
@@ -1,9 +1,9 @@
 [
   {
-    "variant": "mlp-linear-3L",
-    "eval_loss": 3.0161361694335938,
-    "out": "out/mlp-linear-3L_run",
-    "run_name": "LM-mlp-linear-3L-1.0M-20260819-170853",
+    "variant": "mlp-linear-9L",
+    "eval_loss": 2.7597591876983643,
+    "out": "out/mlp-linear-9L_run",
+    "run_name": "LM-mlp-linear-9L-2.0M-20260819-171540",
     "status": "success"
   }
 ]
\ No newline at end of file
diff --git a/zain/Activation/wandb/debug-internal.log b/zain/Activation/wandb/debug-internal.log
index 8721cb52a7e6c028c3628f00e6503a5f097ffc11..2a0f0d5109564c080980d1b9266553a6839105a3 100644
--- a/zain/Activation/wandb/debug-internal.log
+++ b/zain/Activation/wandb/debug-internal.log
@@ -1,23 +1,41 @@
-{"time":"2026-08-19T17:12:15.319602051Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-19T17:12:15.319738433Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-19T17:12:15.582526825Z","level":"INFO","msg":"stream: created new stream","id":"2nil4rqn"}
-{"time":"2026-08-19T17:12:15.582612182Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-19T17:12:15.582724663Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-19T17:12:15.582734966Z","level":"INFO","msg":"writer: started","stream_id":"2nil4rqn"}
-{"time":"2026-08-19T17:12:15.582765014Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-19T17:12:16.123825436Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-19T17:12:16.292352951Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:12:31.124396387Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":5,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":9,"uploaded_len":2}
-{"time":"2026-08-19T17:12:31.267288408Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:12:46.124002043Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":6,"events_offset":1,"events_lines":2,"console_offset":2,"console_lines":1}
-{"time":"2026-08-19T17:12:46.272227989Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:13:01.124054582Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":6,"events_offset":3,"events_lines":2,"console_offset":8,"console_lines":23}
-{"time":"2026-08-19T17:13:01.286514914Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:13:16.124792397Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":6,"events_offset":5,"events_lines":2,"console_offset":24,"console_lines":1}
-{"time":"2026-08-19T17:13:16.296925544Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:13:31.124130053Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":6,"events_offset":7,"events_lines":2,"console_offset":30,"console_lines":23}
-{"time":"2026-08-19T17:13:31.258077274Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:13:46.124150563Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":6,"events_offset":9,"events_lines":2,"console_offset":46,"console_lines":1}
-{"time":"2026-08-19T17:13:46.270511664Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-19T17:14:01.124348067Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":6,"events_offset":11,"events_lines":2,"console_offset":52,"console_lines":23}
-{"time":"2026-08-19T17:14:01.314421382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:15:41.412064429Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-19T17:15:41.41218811Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-19T17:15:41.674976743Z","level":"INFO","msg":"stream: created new stream","id":"bhq23mh5"}
+{"time":"2026-08-19T17:15:41.675059529Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-19T17:15:41.675177927Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-19T17:15:41.675184514Z","level":"INFO","msg":"writer: started","stream_id":"bhq23mh5"}
+{"time":"2026-08-19T17:15:41.675204094Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-19T17:15:42.093210867Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-19T17:15:42.190260489Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:15:57.093327808Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":5,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":9,"uploaded_len":2}
+{"time":"2026-08-19T17:15:57.208750755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:12.09424474Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":6,"events_offset":1,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-19T17:16:12.254954463Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:27.093741092Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":6,"events_offset":3,"events_lines":2,"console_offset":8,"console_lines":22}
+{"time":"2026-08-19T17:16:27.218480793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:42.09555922Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":4,"events_offset":5,"events_lines":2,"console_offset":24,"console_lines":1}
+{"time":"2026-08-19T17:16:42.264597826Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:57.09386987Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":5,"events_offset":7,"events_lines":2,"console_offset":30,"console_lines":19}
+{"time":"2026-08-19T17:16:57.209800875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:12.094049325Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":5,"events_offset":9,"events_lines":2,"console_offset":46,"console_lines":1}
+{"time":"2026-08-19T17:17:12.252024387Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:27.093470433Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":31,"history_lines":4,"events_offset":11,"events_lines":2,"console_offset":49,"console_lines":15}
+{"time":"2026-08-19T17:17:27.202702289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:42.093533111Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":6,"events_offset":13,"events_lines":2,"console_offset":57,"console_lines":1}
+{"time":"2026-08-19T17:17:42.240909652Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:57.093850562Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":6,"events_offset":15,"events_lines":2,"console_offset":63,"console_lines":23}
+{"time":"2026-08-19T17:17:57.20571167Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:12.094059821Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":6,"events_offset":17,"events_lines":2,"console_offset":79,"console_lines":1}
+{"time":"2026-08-19T17:18:12.283285285Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:27.0939628Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":5,"events_offset":19,"events_lines":2,"console_offset":85,"console_lines":21}
+{"time":"2026-08-19T17:18:27.220681676Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:42.09374958Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":3,"events_offset":21,"events_lines":2,"console_offset":101,"console_lines":1}
+{"time":"2026-08-19T17:18:42.218715049Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:45.014142777Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-19T17:18:45.014366349Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":1,"events_offset":23,"events_lines":1,"console_offset":106,"console_lines":15,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-19T17:18:45.153108654Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:45.154305649Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-19T17:18:45.157232952Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-19T17:18:45.157265556Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-19T17:18:45.161743858Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-19T17:18:45.161763043Z","level":"INFO","msg":"stream: all finished"}
diff --git a/zain/Activation/wandb/debug.log b/zain/Activation/wandb/debug.log
index 626869e9106f4b0cb0694b1a8c7e22b937dbca69..43a6eb9157b6147439b6842e8073e28ea86140e2 100644
--- a/zain/Activation/wandb/debug.log
+++ b/zain/Activation/wandb/debug.log
@@ -1,23 +1,28 @@
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_setup.py:_flush():81] Configure stats pid to 3920475
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug.log
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-internal.log
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:init():772] calling init triggers
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_setup.py:_flush():81] Configure stats pid to 3953242
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug.log
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-internal.log
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():772] calling init triggers
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:init():820] starting backend
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
-2026-08-19 17:12:15,318 INFO    MainThread:3920475 [wandb_init.py:init():835] sending inform_init request
-2026-08-19 17:12:15,583 INFO    MainThread:3920475 [wandb_init.py:init():840] backend started and connected
-2026-08-19 17:12:15,587 INFO    MainThread:3920475 [wandb_init.py:init():910] updated telemetry
-2026-08-19 17:12:15,594 INFO    MainThread:3920475 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-19 17:12:16,043 INFO    MainThread:3920475 [wandb_init.py:init():978] starting run threads in backend
-2026-08-19 17:12:16,117 INFO    MainThread:3920475 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-19 17:12:16,117 INFO    MainThread:3920475 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-19 17:12:16,117 INFO    MainThread:3920475 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-19 17:12:16,117 INFO    MainThread:3920475 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-19 17:12:16,120 INFO    MainThread:3920475 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-19 17:12:16,120 INFO    MainThread:3920475 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 3, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'linear', 'waleed_beta': 10.0, 'powlu_m': 3.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-linear-3L_run', 'per_device_train_batch_size': 80, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-linear-3L-1.0M-20260819-171214', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 100, 'eval_delay': 0, 'per_device_eval_batch_size': 1500, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-linear-3L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-19 17:12:16,122 INFO    MainThread:3920475 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 1016704 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14f219615650>>
-2026-08-19 17:12:16,122 INFO    MainThread:3920475 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 1016704 None
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():820] starting backend
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():835] sending inform_init request
+2026-08-19 17:15:41,675 INFO    MainThread:3953242 [wandb_init.py:init():840] backend started and connected
+2026-08-19 17:15:41,679 INFO    MainThread:3953242 [wandb_init.py:init():910] updated telemetry
+2026-08-19 17:15:41,686 INFO    MainThread:3953242 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-19 17:15:42,011 INFO    MainThread:3953242 [wandb_init.py:init():978] starting run threads in backend
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-19 17:15:42,089 INFO    MainThread:3953242 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-19 17:15:42,090 INFO    MainThread:3953242 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 9, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'linear', 'waleed_beta': 10.0, 'powlu_m': 3.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-linear-9L_run', 'per_device_train_batch_size': 80, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-linear-9L-2.0M-20260819-171540', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 100, 'eval_delay': 0, 'per_device_eval_batch_size': 1500, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-linear-9L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-19 17:15:42,091 INFO    MainThread:3953242 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 2001280 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x15079cfad610>>
+2026-08-19 17:15:42,091 INFO    MainThread:3953242 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 2001280 None
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research-ultimate-checking/bhq23mh5
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_restore():2570] restore
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_restore():2576] restore done
+2026-08-19 17:18:45,156 INFO    MainThread:3953242 [wandb_run.py:_footer_sync_info():3993] logging synced files
diff --git a/zain/Activation/wandb/run-20260819_163845-74tq2syl/logs/debug-core.log b/zain/Activation/wandb/run-20260819_163845-74tq2syl/logs/debug-core.log
index 02fe8c0b190d0657c25794035f3ba9d448c905d3..d756776b9e3c29738a5cbd82b38d1311ed0832e0 100644
--- a/zain/Activation/wandb/run-20260819_163845-74tq2syl/logs/debug-core.log
+++ b/zain/Activation/wandb/run-20260819_163845-74tq2syl/logs/debug-core.log
@@ -54,3 +54,23 @@
 {"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}
+{"time":"2026-08-19T17:15:41.262714425Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.411904785Z","level":"INFO","msg":"handleInformInit: received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.67518661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:47.115142635Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:44.577098246Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.155577147Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.157199333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:45.162184308Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.12414285Z","level":"INFO","msg":"processOutgoingData: finished","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124138531Z","level":"INFO","msg":"connection: closing","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124233886Z","level":"INFO","msg":"connection: closed successfully","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124238443Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"7(@)"}
diff --git a/zain/Activation/wandb/run-20260819_164534-glh82iyh/logs/debug-core.log b/zain/Activation/wandb/run-20260819_164534-glh82iyh/logs/debug-core.log
index 02fe8c0b190d0657c25794035f3ba9d448c905d3..d756776b9e3c29738a5cbd82b38d1311ed0832e0 100644
--- a/zain/Activation/wandb/run-20260819_164534-glh82iyh/logs/debug-core.log
+++ b/zain/Activation/wandb/run-20260819_164534-glh82iyh/logs/debug-core.log
@@ -54,3 +54,23 @@
 {"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}
+{"time":"2026-08-19T17:15:41.262714425Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.411904785Z","level":"INFO","msg":"handleInformInit: received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.67518661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:47.115142635Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:44.577098246Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.155577147Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.157199333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:45.162184308Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.12414285Z","level":"INFO","msg":"processOutgoingData: finished","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124138531Z","level":"INFO","msg":"connection: closing","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124233886Z","level":"INFO","msg":"connection: closed successfully","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124238443Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"7(@)"}
diff --git a/zain/Activation/wandb/run-20260819_164553-44x03ghu/logs/debug-core.log b/zain/Activation/wandb/run-20260819_164553-44x03ghu/logs/debug-core.log
index 02fe8c0b190d0657c25794035f3ba9d448c905d3..d756776b9e3c29738a5cbd82b38d1311ed0832e0 100644
--- a/zain/Activation/wandb/run-20260819_164553-44x03ghu/logs/debug-core.log
+++ b/zain/Activation/wandb/run-20260819_164553-44x03ghu/logs/debug-core.log
@@ -54,3 +54,23 @@
 {"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}
+{"time":"2026-08-19T17:15:41.262714425Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.411904785Z","level":"INFO","msg":"handleInformInit: received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.67518661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:47.115142635Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:44.577098246Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.155577147Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.157199333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:45.162184308Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.12414285Z","level":"INFO","msg":"processOutgoingData: finished","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124138531Z","level":"INFO","msg":"connection: closing","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124233886Z","level":"INFO","msg":"connection: closed successfully","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124238443Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"7(@)"}
diff --git a/zain/Activation/wandb/run-20260819_170854-zhg4u13t/logs/debug-core.log b/zain/Activation/wandb/run-20260819_170854-zhg4u13t/logs/debug-core.log
index 02fe8c0b190d0657c25794035f3ba9d448c905d3..d756776b9e3c29738a5cbd82b38d1311ed0832e0 100644
--- a/zain/Activation/wandb/run-20260819_170854-zhg4u13t/logs/debug-core.log
+++ b/zain/Activation/wandb/run-20260819_170854-zhg4u13t/logs/debug-core.log
@@ -54,3 +54,23 @@
 {"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}
+{"time":"2026-08-19T17:15:41.262714425Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.411904785Z","level":"INFO","msg":"handleInformInit: received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.67518661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:47.115142635Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:44.577098246Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.155577147Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.157199333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:45.162184308Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.12414285Z","level":"INFO","msg":"processOutgoingData: finished","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124138531Z","level":"INFO","msg":"connection: closing","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124233886Z","level":"INFO","msg":"connection: closed successfully","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124238443Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"7(@)"}
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/config.yaml b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0213e1b40b2c2f202557366188ae487e649f7505
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/config.yaml
@@ -0,0 +1,435 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            rjsr5aiivqlwnjxtkxlgjkrgdfxgvs5f:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100L.yaml
+                    - --variants
+                    - mlp-linear-3L
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "708477448192"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 463c9961366755fc55f02df9a0d471b3cbcb025e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-19T17:12:15.316941Z"
+                writerId: rjsr5aiivqlwnjxtkxlgjkrgdfxgvs5f
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: linear
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 100
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-mlp-linear-3L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: mlp
+model/num_parameters:
+    value: 1016704
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 3
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/mlp-linear-3L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 1500
+per_device_train_batch_size:
+    value: 80
+powlu_m:
+    value: 3
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-mlp-linear-3L-1.0M-20260819-171214
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/output.log b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/output.log
index 5da5b2b2dae50719b11604fc2e3366dd134bd4f1..f2305eefa7977538e65ef890bfc91cca6826424c 100644
--- a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/output.log
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/output.log
@@ -77,10 +77,45 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 96.
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 340.25it/s]
- 80%|████████  | 800/1000 [01:55<00:17, 11.65it/s]?, ?it/s]
+ 80%|████████  | 800/1000 [02:03<00:17, 11.65[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.99', 'grad_norm': '0.9062', 'learning_rate': '0.001', 'epoch': '0.06067', 'train/total_time_seconds': '12.76', 'train/time_per_step_avg': '0.01649', 'train/epoch_time_elapsed': '109.4', 'train/estimated_remaining_minutes': '0.08272'}
 {'loss': '2.984', 'grad_norm': '1.195', 'learning_rate': '0.001', 'epoch': '0.06235', 'train/total_time_seconds': '13.09', 'train/time_per_step_avg': '0.01648', 'train/epoch_time_elapsed': '110.9', 'train/estimated_remaining_minutes': '0.07666'}
 {'loss': '2.957', 'grad_norm': '1.133', 'learning_rate': '0.001', 'epoch': '0.06404', 'train/total_time_seconds': '13.42', 'train/time_per_step_avg': '0.01648', 'train/epoch_time_elapsed': '112.4', 'train/estimated_remaining_minutes': '0.07064'}
 {'loss': '2.945', 'grad_norm': '0.7539', 'learning_rate': '0.001', 'epoch': '0.06572', 'train/total_time_seconds': '13.75', 'train/time_per_step_avg': '0.01647', 'train/epoch_time_elapsed': '113.9', 'train/estimated_remaining_minutes': '0.06464'}
 {'loss': '2.954', 'grad_norm': '0.8867', 'learning_rate': '0.001', 'epoch': '0.06741', 'train/total_time_seconds': '14.08', 'train/time_per_step_avg': '0.01649', 'train/epoch_time_elapsed': '115.6', 'train/estimated_remaining_minutes': '0.05867'}
- 29%|██▊       | 2/7 [00:01<00:03,  1.66it/s]
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.937', 'eval_runtime': '7.515', 'eval_samples_per_second': '1268', 'eval_steps_per_second': '0.931', 'epoch': '0.06741', 'train/total_time_seconds': '14.08', 'train/time_per_step_avg': '0.01649', 'train/epoch_time_elapsed': '123.1', 'train/estimated_remaining_minutes': '0.05867'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 338.44it/s]
+ 90%|█████████ | 900/1000 [02:18<00:07, 13.35[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.927', 'grad_norm': '0.7422', 'learning_rate': '0.001', 'epoch': '0.06909', 'train/total_time_seconds': '14.41', 'train/time_per_step_avg': '0.01651', 'train/epoch_time_elapsed': '124.6', 'train/estimated_remaining_minutes': '0.05273'}
+{'loss': '2.923', 'grad_norm': '0.8828', 'learning_rate': '0.001', 'epoch': '0.07078', 'train/total_time_seconds': '14.74', 'train/time_per_step_avg': '0.01652', 'train/epoch_time_elapsed': '126.2', 'train/estimated_remaining_minutes': '0.04681'}
+{'loss': '2.909', 'grad_norm': '0.8008', 'learning_rate': '0.001', 'epoch': '0.07246', 'train/total_time_seconds': '15.08', 'train/time_per_step_avg': '0.01655', 'train/epoch_time_elapsed': '127.7', 'train/estimated_remaining_minutes': '0.04091'}
+{'loss': '2.907', 'grad_norm': '1.133', 'learning_rate': '0.001', 'epoch': '0.07415', 'train/total_time_seconds': '15.41', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '129.2', 'train/estimated_remaining_minutes': '0.03502'}
+{'loss': '2.868', 'grad_norm': '0.9922', 'learning_rate': '0.001', 'epoch': '0.07583', 'train/total_time_seconds': '15.74', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '130.7', 'train/estimated_remaining_minutes': '0.02914'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.89', 'eval_runtime': '7.735', 'eval_samples_per_second': '1232', 'eval_steps_per_second': '0.905', 'epoch': '0.07583', 'train/total_time_seconds': '15.74', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '138.4', 'train/estimated_remaining_minutes': '0.02914'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 326.63it/s]
+100%|██████████| 1000/1000 [02:33<00:00, 11.6[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.875', 'grad_norm': '0.8125', 'learning_rate': '0.001', 'epoch': '0.07752', 'train/total_time_seconds': '16.07', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '140', 'train/estimated_remaining_minutes': '0.02329'}
+{'loss': '2.869', 'grad_norm': '0.8047', 'learning_rate': '0.001', 'epoch': '0.0792', 'train/total_time_seconds': '16.4', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '141.5', 'train/estimated_remaining_minutes': '0.01745'}
+{'loss': '2.866', 'grad_norm': '0.8867', 'learning_rate': '0.001', 'epoch': '0.08089', 'train/total_time_seconds': '16.73', 'train/time_per_step_avg': '0.01653', 'train/epoch_time_elapsed': '143', 'train/estimated_remaining_minutes': '0.01162'}
+{'loss': '2.861', 'grad_norm': '0.7383', 'learning_rate': '0.001', 'epoch': '0.08257', 'train/total_time_seconds': '17.06', 'train/time_per_step_avg': '0.01653', 'train/epoch_time_elapsed': '144.5', 'train/estimated_remaining_minutes': '0.005803'}
+{'loss': '2.839', 'grad_norm': '0.8867', 'learning_rate': '0.001', 'epoch': '0.08426', 'train/total_time_seconds': '17.39', 'train/time_per_step_avg': '0.01654', 'train/epoch_time_elapsed': '146', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.841', 'eval_runtime': '7.595', 'eval_samples_per_second': '1254', 'eval_steps_per_second': '0.922', 'epoch': '0.08426', 'train/total_time_seconds': '17.39', 'train/time_per_step_avg': '0.01654', 'train/epoch_time_elapsed': '153.6', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 184.90it/s]
+100%|██████████| 1000/1000 [02:33<00:00,  6.51it/s], ?it/s]
+{'train_runtime': '154.6', 'train_samples_per_second': '517.5', 'train_steps_per_second': '6.468', 'train_loss': '3.728', 'epoch': '0.08426', 'train/total_time_seconds': '17.39', 'train/time_per_step_avg': '0.01654', 'train/epoch_time_elapsed': '153.7', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 7/7 [00:05<00:00,  1.32it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 324.94it/s]
+>>> FINISHED mlp-linear-3L successfully
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/wandb-summary.json b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..dac715a1a4b9957eb78c3913be3788b75b63c0da
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/wandb-summary.json
@@ -0,0 +1 @@
+{"train_steps_per_second":6.468,"_wandb":{"runtime":161},"train/learning_rate":0.001,"train/global_step":1000,"eval/steps_per_second":0.927,"train/epoch":0.08426019548365352,"_timestamp":1.787159697355621e+09,"train_runtime":154.6026,"_runtime":161,"train/loss":2.8389766693115233,"train/grad_norm":0.88671875,"eval/loss":2.841240167617798,"eval/runtime":7.5505,"train_loss":3.7283405075073244,"eval/samples_per_second":1261.772,"_step":61,"total_flos":1.2101615616e+14,"train_samples_per_second":517.456}
\ No newline at end of file
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-core.log b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-core.log
index 02fe8c0b190d0657c25794035f3ba9d448c905d3..d756776b9e3c29738a5cbd82b38d1311ed0832e0 100644
--- a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-core.log
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-core.log
@@ -54,3 +54,23 @@
 {"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}
+{"time":"2026-08-19T17:15:41.262714425Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.411904785Z","level":"INFO","msg":"handleInformInit: received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.67518661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:47.115142635Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:44.577098246Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.155577147Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.157199333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:45.162184308Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.12414285Z","level":"INFO","msg":"processOutgoingData: finished","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124138531Z","level":"INFO","msg":"connection: closing","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124233886Z","level":"INFO","msg":"connection: closed successfully","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124238443Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"7(@)"}
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-internal.log b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-internal.log
index 9213e932847c152276acce3795a6c9427a51a4ae..fe4e9d47f5ebbd75cd0bbc4afd3fde0453635163 100644
--- a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-internal.log
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-internal.log
@@ -22,3 +22,16 @@
 {"time":"2026-08-19T17:14:01.124348067Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":6,"events_offset":11,"events_lines":2,"console_offset":52,"console_lines":23}
 {"time":"2026-08-19T17:14:01.314421382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-19T17:14:16.125013671Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":6,"events_offset":13,"events_lines":2,"console_offset":68,"console_lines":1}
+{"time":"2026-08-19T17:14:16.73098493Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:31.124676234Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":6,"events_offset":15,"events_lines":2,"console_offset":74,"console_lines":23}
+{"time":"2026-08-19T17:14:31.25448083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:46.124973259Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":6,"events_offset":17,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-19T17:14:46.269794228Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:58.136247949Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-19T17:14:58.136517121Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":3,"events_offset":19,"events_lines":1,"console_offset":96,"console_lines":25,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-19T17:14:58.277374581Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:58.278584446Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-19T17:14:58.281745009Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-19T17:14:58.281775882Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-19T17:14:58.281829879Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-19T17:14:58.281833448Z","level":"INFO","msg":"stream: all finished"}
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug.log b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug.log
index 626869e9106f4b0cb0694b1a8c7e22b937dbca69..799c1f341e9657e84e2297ea6ca56f784856217c 100644
--- a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug.log
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug.log
@@ -21,3 +21,8 @@ config: {'_wandb': {}}
 2026-08-19 17:12:16,120 INFO    MainThread:3920475 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 3, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'linear', 'waleed_beta': 10.0, 'powlu_m': 3.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-linear-3L_run', 'per_device_train_batch_size': 80, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-linear-3L-1.0M-20260819-171214', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 100, 'eval_delay': 0, 'per_device_eval_batch_size': 1500, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-linear-3L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-19 17:12:16,122 INFO    MainThread:3920475 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 1016704 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14f219615650>>
 2026-08-19 17:12:16,122 INFO    MainThread:3920475 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 1016704 None
+2026-08-19 17:14:57,366 INFO    MainThread:3920475 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research-ultimate-checking/2nil4rqn
+2026-08-19 17:14:57,367 INFO    MainThread:3920475 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-19 17:14:57,367 INFO    MainThread:3920475 [wandb_run.py:_restore():2570] restore
+2026-08-19 17:14:57,367 INFO    MainThread:3920475 [wandb_run.py:_restore():2576] restore done
+2026-08-19 17:14:58,281 INFO    MainThread:3920475 [wandb_run.py:_footer_sync_info():3993] logging synced files
diff --git a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/run-2nil4rqn.wandb b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/run-2nil4rqn.wandb
index 821f8323f60721c5eb23acc936314b46e96f608e..33d880c93bd2af8b945d703f1b7f6bc8189b282d 100644
--- a/zain/Activation/wandb/run-20260819_171215-2nil4rqn/run-2nil4rqn.wandb
+++ b/zain/Activation/wandb/run-20260819_171215-2nil4rqn/run-2nil4rqn.wandb
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3de8ec357a60a3d84aa75665c9087e461ce8da0ae326066fb9c12e56d1f75ef0
-size 163840
+oid sha256:a7b5da609b9fccdc3e7dd6fde9d7d49ec5a9b66eb9fe3dc139ddf206bfab728d
+size 236528
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/config.yaml b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..17947266d91dedc0fef58fb80ac32ef1f2cdbdd2
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/config.yaml
@@ -0,0 +1,435 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            2l4ndnmtei7frfwveku3d762xxak1bdj:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100L.yaml
+                    - --variants
+                    - mlp-linear-9L
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "708485218304"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 463c9961366755fc55f02df9a0d471b3cbcb025e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-19T17:15:41.408834Z"
+                writerId: 2l4ndnmtei7frfwveku3d762xxak1bdj
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: linear
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 100
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-mlp-linear-9L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: mlp
+model/num_parameters:
+    value: 2001280
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 9
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/mlp-linear-9L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 1500
+per_device_train_batch_size:
+    value: 80
+powlu_m:
+    value: 3
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-mlp-linear-9L-2.0M-20260819-171540
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/output.log b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..ffe4c6d9fee17bc6d0422d7ccad79fb012d4715c
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/output.log
@@ -0,0 +1,121 @@
+  0%|          | 0/1000 [00:00<?, ?it/s][transformers] `use_return_dict` is deprecated! Use `return_dict` instead!
+[INFO] Causal mask (float with -inf) applied to all attention layers.
+ 10%|█         | 100/1000 [00:18<01:34,  9.55[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.201', 'grad_norm': '1.242', 'learning_rate': '9.5e-05', 'epoch': '0.001685', 'train/total_time_seconds': '1.182', 'train/time_per_step_avg': '0.0591', 'train/epoch_time_elapsed': '2.652', 'train/estimated_remaining_minutes': '0.9653'}
+{'loss': '7.765', 'grad_norm': '1.242', 'learning_rate': '0.000195', 'epoch': '0.00337', 'train/total_time_seconds': '1.819', 'train/time_per_step_avg': '0.04547', 'train/epoch_time_elapsed': '4.584', 'train/estimated_remaining_minutes': '0.7275'}
+{'loss': '7.16', 'grad_norm': '1.227', 'learning_rate': '0.000295', 'epoch': '0.005056', 'train/total_time_seconds': '2.454', 'train/time_per_step_avg': '0.0409', 'train/epoch_time_elapsed': '6.432', 'train/estimated_remaining_minutes': '0.6408'}
+{'loss': '6.481', 'grad_norm': '1.211', 'learning_rate': '0.000395', 'epoch': '0.006741', 'train/total_time_seconds': '3.086', 'train/time_per_step_avg': '0.03858', 'train/epoch_time_elapsed': '8.246', 'train/estimated_remaining_minutes': '0.5916'}
+{'loss': '5.919', 'grad_norm': '1.008', 'learning_rate': '0.000495', 'epoch': '0.008426', 'train/total_time_seconds': '3.722', 'train/time_per_step_avg': '0.03722', 'train/epoch_time_elapsed': '10.15', 'train/estimated_remaining_minutes': '0.5584'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '5.634', 'eval_runtime': '7.97', 'eval_samples_per_second': '1195', 'eval_steps_per_second': '0.878', 'epoch': '0.008426', 'train/total_time_seconds': '3.722', 'train/time_per_step_avg': '0.03722', 'train/epoch_time_elapsed': '18.12', 'train/estimated_remaining_minutes': '0.5584'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 93.80it/s]
+ 20%|██        | 200/1000 [00:35<01:13, 10.84[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '5.413', 'grad_norm': '1.172', 'learning_rate': '0.000595', 'epoch': '0.01011', 'train/total_time_seconds': '4.364', 'train/time_per_step_avg': '0.03182', 'train/epoch_time_elapsed': '20.01', 'train/estimated_remaining_minutes': '0.5334'}
+{'loss': '5.033', 'grad_norm': '1.516', 'learning_rate': '0.000695', 'epoch': '0.0118', 'train/total_time_seconds': '5.002', 'train/time_per_step_avg': '0.03183', 'train/epoch_time_elapsed': '21.84', 'train/estimated_remaining_minutes': '0.5121'}
+{'loss': '4.695', 'grad_norm': '0.9766', 'learning_rate': '0.000795', 'epoch': '0.01348', 'train/total_time_seconds': '5.642', 'train/time_per_step_avg': '0.03187', 'train/epoch_time_elapsed': '23.68', 'train/estimated_remaining_minutes': '0.4936'}
+{'loss': '4.425', 'grad_norm': '0.8828', 'learning_rate': '0.000895', 'epoch': '0.01517', 'train/total_time_seconds': '6.286', 'train/time_per_step_avg': '0.032', 'train/epoch_time_elapsed': '25.54', 'train/estimated_remaining_minutes': '0.4773'}
+{'loss': '4.205', 'grad_norm': '0.8789', 'learning_rate': '0.000995', 'epoch': '0.01685', 'train/total_time_seconds': '6.934', 'train/time_per_step_avg': '0.03212', 'train/epoch_time_elapsed': '27.38', 'train/estimated_remaining_minutes': '0.4623'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '4.133', 'eval_runtime': '7.933', 'eval_samples_per_second': '1201', 'eval_steps_per_second': '0.882', 'epoch': '0.01685', 'train/total_time_seconds': '6.934', 'train/time_per_step_avg': '0.03212', 'train/epoch_time_elapsed': '35.32', 'train/estimated_remaining_minutes': '0.4623'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 114.05it/s]
+ 30%|███       | 300/1000 [00:52<01:04, 10.91[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '4.049', 'grad_norm': '0.6875', 'learning_rate': '0.001', 'epoch': '0.01854', 'train/total_time_seconds': '7.582', 'train/time_per_step_avg': '0.03218', 'train/epoch_time_elapsed': '37.19', 'train/estimated_remaining_minutes': '0.448'}
+{'loss': '3.919', 'grad_norm': '0.7383', 'learning_rate': '0.001', 'epoch': '0.02022', 'train/total_time_seconds': '8.228', 'train/time_per_step_avg': '0.03226', 'train/epoch_time_elapsed': '39.05', 'train/estimated_remaining_minutes': '0.4343'}
+{'loss': '3.783', 'grad_norm': '0.6836', 'learning_rate': '0.001', 'epoch': '0.02191', 'train/total_time_seconds': '8.873', 'train/time_per_step_avg': '0.03231', 'train/epoch_time_elapsed': '40.89', 'train/estimated_remaining_minutes': '0.4209'}
+{'loss': '3.701', 'grad_norm': '0.8242', 'learning_rate': '0.001', 'epoch': '0.02359', 'train/total_time_seconds': '9.516', 'train/time_per_step_avg': '0.0323', 'train/epoch_time_elapsed': '42.82', 'train/estimated_remaining_minutes': '0.4078'}
+{'loss': '3.636', 'grad_norm': '0.9844', 'learning_rate': '0.001', 'epoch': '0.02528', 'train/total_time_seconds': '10.16', 'train/time_per_step_avg': '0.03223', 'train/epoch_time_elapsed': '44.65', 'train/estimated_remaining_minutes': '0.395'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '3.598', 'eval_runtime': '7.973', 'eval_samples_per_second': '1195', 'eval_steps_per_second': '0.878', 'epoch': '0.02528', 'train/total_time_seconds': '10.16', 'train/time_per_step_avg': '0.03223', 'train/epoch_time_elapsed': '52.63', 'train/estimated_remaining_minutes': '0.395'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 170.80it/s]
+ 40%|████      | 400/1000 [01:10<00:56, 10.56[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '3.539', 'grad_norm': '0.8086', 'learning_rate': '0.001', 'epoch': '0.02696', 'train/total_time_seconds': '10.8', 'train/time_per_step_avg': '0.03215', 'train/epoch_time_elapsed': '54.51', 'train/estimated_remaining_minutes': '0.3824'}
+{'loss': '3.492', 'grad_norm': '0.7227', 'learning_rate': '0.001', 'epoch': '0.02865', 'train/total_time_seconds': '11.44', 'train/time_per_step_avg': '0.03212', 'train/epoch_time_elapsed': '56.36', 'train/estimated_remaining_minutes': '0.3701'}
+{'loss': '3.443', 'grad_norm': '0.8203', 'learning_rate': '0.001', 'epoch': '0.03033', 'train/total_time_seconds': '12.08', 'train/time_per_step_avg': '0.03205', 'train/epoch_time_elapsed': '58.19', 'train/estimated_remaining_minutes': '0.3579'}
+{'loss': '3.399', 'grad_norm': '0.8203', 'learning_rate': '0.001', 'epoch': '0.03202', 'train/total_time_seconds': '12.72', 'train/time_per_step_avg': '0.03201', 'train/epoch_time_elapsed': '60.08', 'train/estimated_remaining_minutes': '0.3458'}
+{'loss': '3.342', 'grad_norm': '0.6484', 'learning_rate': '0.001', 'epoch': '0.0337', 'train/total_time_seconds': '13.36', 'train/time_per_step_avg': '0.03201', 'train/epoch_time_elapsed': '61.94', 'train/estimated_remaining_minutes': '0.334'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '3.33', 'eval_runtime': '8.169', 'eval_samples_per_second': '1166', 'eval_steps_per_second': '0.857', 'epoch': '0.0337', 'train/total_time_seconds': '13.36', 'train/time_per_step_avg': '0.03201', 'train/epoch_time_elapsed': '70.11', 'train/estimated_remaining_minutes': '0.334'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 115.56it/s]
+ 50%|█████     | 500/1000 [01:27<00:46, 10.76[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '3.305', 'grad_norm': '0.7109', 'learning_rate': '0.001', 'epoch': '0.03539', 'train/total_time_seconds': '14', 'train/time_per_step_avg': '0.03204', 'train/epoch_time_elapsed': '72.01', 'train/estimated_remaining_minutes': '0.3222'}
+{'loss': '3.259', 'grad_norm': '0.7188', 'learning_rate': '0.001', 'epoch': '0.03707', 'train/total_time_seconds': '14.65', 'train/time_per_step_avg': '0.03212', 'train/epoch_time_elapsed': '73.95', 'train/estimated_remaining_minutes': '0.3108'}
+{'loss': '3.229', 'grad_norm': '0.7305', 'learning_rate': '0.001', 'epoch': '0.03876', 'train/total_time_seconds': '15.3', 'train/time_per_step_avg': '0.0322', 'train/epoch_time_elapsed': '75.83', 'train/estimated_remaining_minutes': '0.2993'}
+{'loss': '3.202', 'grad_norm': '0.7031', 'learning_rate': '0.001', 'epoch': '0.04044', 'train/total_time_seconds': '15.94', 'train/time_per_step_avg': '0.03219', 'train/epoch_time_elapsed': '77.68', 'train/estimated_remaining_minutes': '0.2877'}
+{'loss': '3.164', 'grad_norm': '0.7266', 'learning_rate': '0.001', 'epoch': '0.04213', 'train/total_time_seconds': '16.57', 'train/time_per_step_avg': '0.03212', 'train/epoch_time_elapsed': '79.6', 'train/estimated_remaining_minutes': '0.2762'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '3.157', 'eval_runtime': '7.95', 'eval_samples_per_second': '1198', 'eval_steps_per_second': '0.88', 'epoch': '0.04213', 'train/total_time_seconds': '16.57', 'train/time_per_step_avg': '0.03212', 'train/epoch_time_elapsed': '87.56', 'train/estimated_remaining_minutes': '0.2762'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 173.21it/s]
+ 60%|██████    | 600/1000 [01:45<00:39, 10.00[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '3.145', 'grad_norm': '0.6953', 'learning_rate': '0.001', 'epoch': '0.04382', 'train/total_time_seconds': '17.2', 'train/time_per_step_avg': '0.03203', 'train/epoch_time_elapsed': '89.41', 'train/estimated_remaining_minutes': '0.2647'}
+{'loss': '3.104', 'grad_norm': '0.8281', 'learning_rate': '0.001', 'epoch': '0.0455', 'train/total_time_seconds': '17.84', 'train/time_per_step_avg': '0.03186', 'train/epoch_time_elapsed': '91.23', 'train/estimated_remaining_minutes': '0.2533'}
+{'loss': '3.084', 'grad_norm': '0.7578', 'learning_rate': '0.001', 'epoch': '0.04719', 'train/total_time_seconds': '18.47', 'train/time_per_step_avg': '0.03174', 'train/epoch_time_elapsed': '93.06', 'train/estimated_remaining_minutes': '0.2419'}
+{'loss': '3.05', 'grad_norm': '0.7695', 'learning_rate': '0.001', 'epoch': '0.04887', 'train/total_time_seconds': '19.11', 'train/time_per_step_avg': '0.03169', 'train/epoch_time_elapsed': '94.88', 'train/estimated_remaining_minutes': '0.2306'}
+{'loss': '3.038', 'grad_norm': '0.9688', 'learning_rate': '0.001', 'epoch': '0.05056', 'train/total_time_seconds': '19.75', 'train/time_per_step_avg': '0.0318', 'train/epoch_time_elapsed': '96.94', 'train/estimated_remaining_minutes': '0.2195'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '3.031', 'eval_runtime': '8.12', 'eval_samples_per_second': '1173', 'eval_steps_per_second': '0.862', 'epoch': '0.05056', 'train/total_time_seconds': '19.75', 'train/time_per_step_avg': '0.0318', 'train/epoch_time_elapsed': '105.1', 'train/estimated_remaining_minutes': '0.2195'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 172.53it/s]
+ 70%|███████   | 700/1000 [02:02<00:27, 11.05[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '3.031', 'grad_norm': '0.8242', 'learning_rate': '0.001', 'epoch': '0.05224', 'train/total_time_seconds': '20.38', 'train/time_per_step_avg': '0.03179', 'train/epoch_time_elapsed': '106.9', 'train/estimated_remaining_minutes': '0.2082'}
+{'loss': '3.001', 'grad_norm': '0.6406', 'learning_rate': '0.001', 'epoch': '0.05393', 'train/total_time_seconds': '21.01', 'train/time_per_step_avg': '0.03175', 'train/epoch_time_elapsed': '108.7', 'train/estimated_remaining_minutes': '0.197'}
+{'loss': '2.996', 'grad_norm': '0.707', 'learning_rate': '0.001', 'epoch': '0.05561', 'train/total_time_seconds': '21.64', 'train/time_per_step_avg': '0.03173', 'train/epoch_time_elapsed': '110.6', 'train/estimated_remaining_minutes': '0.1858'}
+{'loss': '2.957', 'grad_norm': '0.7109', 'learning_rate': '0.001', 'epoch': '0.0573', 'train/total_time_seconds': '22.27', 'train/time_per_step_avg': '0.03168', 'train/epoch_time_elapsed': '112.4', 'train/estimated_remaining_minutes': '0.1747'}
+{'loss': '2.94', 'grad_norm': '0.6992', 'learning_rate': '0.001', 'epoch': '0.05898', 'train/total_time_seconds': '22.9', 'train/time_per_step_avg': '0.03153', 'train/epoch_time_elapsed': '114.2', 'train/estimated_remaining_minutes': '0.1636'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.939', 'eval_runtime': '7.975', 'eval_samples_per_second': '1195', 'eval_steps_per_second': '0.878', 'epoch': '0.05898', 'train/total_time_seconds': '22.9', 'train/time_per_step_avg': '0.03153', 'train/epoch_time_elapsed': '122.2', 'train/estimated_remaining_minutes': '0.1636'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 171.02it/s]
+ 80%|████████  | 800/1000 [02:19<00:18, 10.93[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.923', 'grad_norm': '0.6484', 'learning_rate': '0.001', 'epoch': '0.06067', 'train/total_time_seconds': '23.54', 'train/time_per_step_avg': '0.03154', 'train/epoch_time_elapsed': '124.1', 'train/estimated_remaining_minutes': '0.1526'}
+{'loss': '2.915', 'grad_norm': '0.957', 'learning_rate': '0.001', 'epoch': '0.06235', 'train/total_time_seconds': '24.17', 'train/time_per_step_avg': '0.03162', 'train/epoch_time_elapsed': '125.9', 'train/estimated_remaining_minutes': '0.1416'}
+{'loss': '2.888', 'grad_norm': '0.7109', 'learning_rate': '0.001', 'epoch': '0.06404', 'train/total_time_seconds': '24.81', 'train/time_per_step_avg': '0.03166', 'train/epoch_time_elapsed': '127.9', 'train/estimated_remaining_minutes': '0.1306'}
+{'loss': '2.869', 'grad_norm': '0.6914', 'learning_rate': '0.001', 'epoch': '0.06572', 'train/total_time_seconds': '25.45', 'train/time_per_step_avg': '0.03176', 'train/epoch_time_elapsed': '129.7', 'train/estimated_remaining_minutes': '0.1196'}
+{'loss': '2.879', 'grad_norm': '0.6836', 'learning_rate': '0.001', 'epoch': '0.06741', 'train/total_time_seconds': '26.09', 'train/time_per_step_avg': '0.03185', 'train/epoch_time_elapsed': '131.6', 'train/estimated_remaining_minutes': '0.1087'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.863', 'eval_runtime': '8.277', 'eval_samples_per_second': '1151', 'eval_steps_per_second': '0.846', 'epoch': '0.06741', 'train/total_time_seconds': '26.09', 'train/time_per_step_avg': '0.03185', 'train/epoch_time_elapsed': '139.9', 'train/estimated_remaining_minutes': '0.1087'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 118.50it/s]
+ 90%|█████████ | 900/1000 [02:37<00:09, 10.89[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.85', 'grad_norm': '0.6797', 'learning_rate': '0.001', 'epoch': '0.06909', 'train/total_time_seconds': '26.72', 'train/time_per_step_avg': '0.03185', 'train/epoch_time_elapsed': '141.8', 'train/estimated_remaining_minutes': '0.09776'}
+{'loss': '2.848', 'grad_norm': '0.6211', 'learning_rate': '0.001', 'epoch': '0.07078', 'train/total_time_seconds': '27.36', 'train/time_per_step_avg': '0.03182', 'train/epoch_time_elapsed': '143.6', 'train/estimated_remaining_minutes': '0.08685'}
+{'loss': '2.834', 'grad_norm': '0.7305', 'learning_rate': '0.001', 'epoch': '0.07246', 'train/total_time_seconds': '27.99', 'train/time_per_step_avg': '0.03179', 'train/epoch_time_elapsed': '145.5', 'train/estimated_remaining_minutes': '0.07594'}
+{'loss': '2.825', 'grad_norm': '0.6992', 'learning_rate': '0.001', 'epoch': '0.07415', 'train/total_time_seconds': '28.62', 'train/time_per_step_avg': '0.0317', 'train/epoch_time_elapsed': '147.3', 'train/estimated_remaining_minutes': '0.06505'}
+{'loss': '2.785', 'grad_norm': '0.6875', 'learning_rate': '0.001', 'epoch': '0.07583', 'train/total_time_seconds': '29.25', 'train/time_per_step_avg': '0.03166', 'train/epoch_time_elapsed': '149.1', 'train/estimated_remaining_minutes': '0.05418'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.803', 'eval_runtime': '7.959', 'eval_samples_per_second': '1197', 'eval_steps_per_second': '0.879', 'epoch': '0.07583', 'train/total_time_seconds': '29.25', 'train/time_per_step_avg': '0.03166', 'train/epoch_time_elapsed': '157.1', 'train/estimated_remaining_minutes': '0.05418'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 84.10it/s]
+100%|██████████| 1000/1000 [02:54<00:00, 10.5[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.794', 'grad_norm': '0.6094', 'learning_rate': '0.001', 'epoch': '0.07752', 'train/total_time_seconds': '29.89', 'train/time_per_step_avg': '0.03165', 'train/epoch_time_elapsed': '159.1', 'train/estimated_remaining_minutes': '0.04331'}
+{'loss': '2.789', 'grad_norm': '0.6719', 'learning_rate': '0.001', 'epoch': '0.0792', 'train/total_time_seconds': '30.52', 'train/time_per_step_avg': '0.03165', 'train/epoch_time_elapsed': '161', 'train/estimated_remaining_minutes': '0.03247'}
+{'loss': '2.785', 'grad_norm': '0.582', 'learning_rate': '0.001', 'epoch': '0.08089', 'train/total_time_seconds': '31.16', 'train/time_per_step_avg': '0.03167', 'train/epoch_time_elapsed': '162.8', 'train/estimated_remaining_minutes': '0.02164'}
+{'loss': '2.775', 'grad_norm': '0.6055', 'learning_rate': '0.001', 'epoch': '0.08257', 'train/total_time_seconds': '31.79', 'train/time_per_step_avg': '0.0317', 'train/epoch_time_elapsed': '164.6', 'train/estimated_remaining_minutes': '0.01081'}
+{'loss': '2.754', 'grad_norm': '0.7812', 'learning_rate': '0.001', 'epoch': '0.08426', 'train/total_time_seconds': '32.44', 'train/time_per_step_avg': '0.03184', 'train/epoch_time_elapsed': '166.6', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.76', 'eval_runtime': '7.927', 'eval_samples_per_second': '1202', 'eval_steps_per_second': '0.883', 'epoch': '0.08426', 'train/total_time_seconds': '32.44', 'train/time_per_step_avg': '0.03184', 'train/epoch_time_elapsed': '174.5', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 114.15it/s]
+100%|██████████| 1000/1000 [02:54<00:00,  5.73it/s], ?it/s]
+{'train_runtime': '175.4', 'train_samples_per_second': '456.1', 'train_steps_per_second': '5.701', 'train_loss': '3.692', 'epoch': '0.08426', 'train/total_time_seconds': '32.44', 'train/time_per_step_avg': '0.03184', 'train/epoch_time_elapsed': '174.6', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 7/7 [00:05<00:00,  1.22it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 172.45it/s]
+>>> FINISHED mlp-linear-9L successfully
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/requirements.txt b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ee8cc78fa3966a6ae1d63af7b60a8c5f22ab2d34
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/requirements.txt
@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+tokenizers==0.23.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/wandb-metadata.json b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..b76f52b8e79bb5035e2a52a7bed309e39cdefe8c
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/wandb-metadata.json
@@ -0,0 +1,95 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-19T17:15:41.408834Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100L.yaml",
+    "--variants",
+    "mlp-linear-9L"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "463c9961366755fc55f02df9a0d471b3cbcb025e"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "708485218304"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "2l4ndnmtei7frfwveku3d762xxak1bdj"
+}
\ No newline at end of file
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/wandb-summary.json b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..28a96b2d6c24dd4696d394102ff5ee84607668f4
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/files/wandb-summary.json
@@ -0,0 +1 @@
+{"train/global_step":1000,"train_steps_per_second":5.701,"eval/samples_per_second":1206.607,"train/learning_rate":0.001,"eval/loss":2.7597591876983643,"eval/steps_per_second":0.887,"_runtime":182,"_wandb":{"runtime":182},"_timestamp":1.7871599245612671e+09,"eval/runtime":7.8957,"train/grad_norm":0.78125,"train/epoch":0.08426019548365352,"_step":61,"total_flos":3.6298555392e+14,"train_loss":3.6923015975952147,"train_runtime":175.4068,"train/loss":2.753749656677246,"train_samples_per_second":456.083}
\ No newline at end of file
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-core.log b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..d756776b9e3c29738a5cbd82b38d1311ed0832e0
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-core.log
@@ -0,0 +1,76 @@
+{"time":"2026-08-19T16:38:32.839280488Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpn7u591yp/port-3590110.txt","pid":3590110,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-19T16:38:32.840574816Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3590110}
+{"time":"2026-08-19T16:38:32.84057072Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3590110-3591582-3840970873/socket","Net":"unix"}}
+{"time":"2026-08-19T16:38:33.017601922Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-19T16:38:45.523223978Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"2(@)"}
+{"time":"2026-08-19T16:38:45.593765665Z","level":"INFO","msg":"handleInformInit: received","streamId":"74tq2syl","id":"2(@)"}
+{"time":"2026-08-19T16:38:45.85943264Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"74tq2syl","id":"2(@)"}
+{"time":"2026-08-19T16:38:51.253972428Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"s41hhf712g5d"}
+{"time":"2026-08-19T16:43:55.859246567Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"s41hhf712g5d"}
+{"time":"2026-08-19T16:43:56.562384948Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"s41hhf712g5d"}
+{"time":"2026-08-19T16:43:56.564075521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"74tq2syl","id":"2(@)"}
+{"time":"2026-08-19T16:43:56.56470085Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"74tq2syl","id":"2(@)"}
+{"time":"2026-08-19T16:43:58.189628272Z","level":"INFO","msg":"connection: closing","id":"2(@)"}
+{"time":"2026-08-19T16:43:58.189706084Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
+{"time":"2026-08-19T16:43:58.189644595Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
+{"time":"2026-08-19T16:43:58.189713806Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-19T16:45:34.355733713Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-19T16:45:34.483865974Z","level":"INFO","msg":"handleInformInit: received","streamId":"glh82iyh","id":"3(@)"}
+{"time":"2026-08-19T16:45:34.744778959Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"glh82iyh","id":"3(@)"}
+{"time":"2026-08-19T16:45:40.185561802Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"d1gg1cuklin0"}
+{"time":"2026-08-19T16:45:40.195891916Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"d1gg1cuklin0"}
+{"time":"2026-08-19T16:45:40.960129401Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"d1gg1cuklin0"}
+{"time":"2026-08-19T16:45:40.961603019Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"glh82iyh","id":"3(@)"}
+{"time":"2026-08-19T16:45:40.96217202Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"glh82iyh","id":"3(@)"}
+{"time":"2026-08-19T16:45:42.643090062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-19T16:45:42.643171917Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-19T16:45:42.643096466Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-19T16:45:42.643183903Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}
+{"time":"2026-08-19T16:45:53.775307513Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"4(@)"}
+{"time":"2026-08-19T16:45:53.86012332Z","level":"INFO","msg":"handleInformInit: received","streamId":"44x03ghu","id":"4(@)"}
+{"time":"2026-08-19T16:45:54.119528402Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"44x03ghu","id":"4(@)"}
+{"time":"2026-08-19T16:45:59.764320578Z","level":"INFO","msg":"connection: cancelling request","id":"4(@)","requestId":"u6tmkghavygl"}
+{"time":"2026-08-19T16:50:40.211693707Z","level":"INFO","msg":"connection: cancelling request","id":"4(@)","requestId":"u6tmkghavygl"}
+{"time":"2026-08-19T16:50:40.749569572Z","level":"INFO","msg":"connection: cancelling request","id":"4(@)","requestId":"u6tmkghavygl"}
+{"time":"2026-08-19T16:50:40.751009589Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"44x03ghu","id":"4(@)"}
+{"time":"2026-08-19T16:50:40.751589921Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"44x03ghu","id":"4(@)"}
+{"time":"2026-08-19T16:50:42.667568611Z","level":"INFO","msg":"connection: closing","id":"4(@)"}
+{"time":"2026-08-19T16:50:42.66767627Z","level":"INFO","msg":"connection: closed successfully","id":"4(@)"}
+{"time":"2026-08-19T16:50:42.667582108Z","level":"INFO","msg":"processOutgoingData: finished","id":"4(@)"}
+{"time":"2026-08-19T16:50:42.667687704Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"4(@)"}
+{"time":"2026-08-19T17:08:54.476108098Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"5(@)"}
+{"time":"2026-08-19T17:08:54.716542574Z","level":"INFO","msg":"handleInformInit: received","streamId":"zhg4u13t","id":"5(@)"}
+{"time":"2026-08-19T17:08:54.983870419Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"zhg4u13t","id":"5(@)"}
+{"time":"2026-08-19T17:09:00.342597647Z","level":"INFO","msg":"connection: cancelling request","id":"5(@)","requestId":"qy20ie44zvce"}
+{"time":"2026-08-19T17:11:36.384264189Z","level":"INFO","msg":"connection: cancelling request","id":"5(@)","requestId":"qy20ie44zvce"}
+{"time":"2026-08-19T17:11:36.932623627Z","level":"INFO","msg":"connection: cancelling request","id":"5(@)","requestId":"qy20ie44zvce"}
+{"time":"2026-08-19T17:11:36.934203156Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"zhg4u13t","id":"5(@)"}
+{"time":"2026-08-19T17:11:36.934659413Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"zhg4u13t","id":"5(@)"}
+{"time":"2026-08-19T17:11:38.381994049Z","level":"INFO","msg":"connection: closing","id":"5(@)"}
+{"time":"2026-08-19T17:11:38.382078588Z","level":"INFO","msg":"connection: closed successfully","id":"5(@)"}
+{"time":"2026-08-19T17:11:38.382003656Z","level":"INFO","msg":"processOutgoingData: finished","id":"5(@)"}
+{"time":"2026-08-19T17:11:38.382088954Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"5(@)"}
+{"time":"2026-08-19T17:12:15.205004959Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"6(@)"}
+{"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}
+{"time":"2026-08-19T17:15:41.262714425Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.411904785Z","level":"INFO","msg":"handleInformInit: received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:41.67518661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:15:47.115142635Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:44.577098246Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.155577147Z","level":"INFO","msg":"connection: cancelling request","id":"7(@)","requestId":"dp9grmbn4dao"}
+{"time":"2026-08-19T17:18:45.157199333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:45.162184308Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bhq23mh5","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.12414285Z","level":"INFO","msg":"processOutgoingData: finished","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124138531Z","level":"INFO","msg":"connection: closing","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124233886Z","level":"INFO","msg":"connection: closed successfully","id":"7(@)"}
+{"time":"2026-08-19T17:18:47.124238443Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"7(@)"}
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-internal.log b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..2a0f0d5109564c080980d1b9266553a6839105a3
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-internal.log
@@ -0,0 +1,41 @@
+{"time":"2026-08-19T17:15:41.412064429Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-19T17:15:41.41218811Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-19T17:15:41.674976743Z","level":"INFO","msg":"stream: created new stream","id":"bhq23mh5"}
+{"time":"2026-08-19T17:15:41.675059529Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-19T17:15:41.675177927Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-19T17:15:41.675184514Z","level":"INFO","msg":"writer: started","stream_id":"bhq23mh5"}
+{"time":"2026-08-19T17:15:41.675204094Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-19T17:15:42.093210867Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-19T17:15:42.190260489Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:15:57.093327808Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":5,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":9,"uploaded_len":2}
+{"time":"2026-08-19T17:15:57.208750755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:12.09424474Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":6,"events_offset":1,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-19T17:16:12.254954463Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:27.093741092Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":6,"events_offset":3,"events_lines":2,"console_offset":8,"console_lines":22}
+{"time":"2026-08-19T17:16:27.218480793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:42.09555922Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":4,"events_offset":5,"events_lines":2,"console_offset":24,"console_lines":1}
+{"time":"2026-08-19T17:16:42.264597826Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:16:57.09386987Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":5,"events_offset":7,"events_lines":2,"console_offset":30,"console_lines":19}
+{"time":"2026-08-19T17:16:57.209800875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:12.094049325Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":5,"events_offset":9,"events_lines":2,"console_offset":46,"console_lines":1}
+{"time":"2026-08-19T17:17:12.252024387Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:27.093470433Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":31,"history_lines":4,"events_offset":11,"events_lines":2,"console_offset":49,"console_lines":15}
+{"time":"2026-08-19T17:17:27.202702289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:42.093533111Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":6,"events_offset":13,"events_lines":2,"console_offset":57,"console_lines":1}
+{"time":"2026-08-19T17:17:42.240909652Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:17:57.093850562Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":6,"events_offset":15,"events_lines":2,"console_offset":63,"console_lines":23}
+{"time":"2026-08-19T17:17:57.20571167Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:12.094059821Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":6,"events_offset":17,"events_lines":2,"console_offset":79,"console_lines":1}
+{"time":"2026-08-19T17:18:12.283285285Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:27.0939628Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":5,"events_offset":19,"events_lines":2,"console_offset":85,"console_lines":21}
+{"time":"2026-08-19T17:18:27.220681676Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:42.09374958Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":3,"events_offset":21,"events_lines":2,"console_offset":101,"console_lines":1}
+{"time":"2026-08-19T17:18:42.218715049Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:45.014142777Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-19T17:18:45.014366349Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":1,"events_offset":23,"events_lines":1,"console_offset":106,"console_lines":15,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-19T17:18:45.153108654Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:18:45.154305649Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-19T17:18:45.157232952Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-19T17:18:45.157265556Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-19T17:18:45.161743858Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-19T17:18:45.161763043Z","level":"INFO","msg":"stream: all finished"}
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug.log b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..43a6eb9157b6147439b6842e8073e28ea86140e2
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug.log
@@ -0,0 +1,28 @@
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_setup.py:_flush():81] Configure stats pid to 3953242
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug.log
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260819_171541-bhq23mh5/logs/debug-internal.log
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():772] calling init triggers
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():820] starting backend
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-19 17:15:41,410 INFO    MainThread:3953242 [wandb_init.py:init():835] sending inform_init request
+2026-08-19 17:15:41,675 INFO    MainThread:3953242 [wandb_init.py:init():840] backend started and connected
+2026-08-19 17:15:41,679 INFO    MainThread:3953242 [wandb_init.py:init():910] updated telemetry
+2026-08-19 17:15:41,686 INFO    MainThread:3953242 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-19 17:15:42,011 INFO    MainThread:3953242 [wandb_init.py:init():978] starting run threads in backend
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-19 17:15:42,086 INFO    MainThread:3953242 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-19 17:15:42,089 INFO    MainThread:3953242 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-19 17:15:42,090 INFO    MainThread:3953242 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 9, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'linear', 'waleed_beta': 10.0, 'powlu_m': 3.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-linear-9L_run', 'per_device_train_batch_size': 80, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-linear-9L-2.0M-20260819-171540', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 100, 'eval_delay': 0, 'per_device_eval_batch_size': 1500, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-linear-9L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-19 17:15:42,091 INFO    MainThread:3953242 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 2001280 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x15079cfad610>>
+2026-08-19 17:15:42,091 INFO    MainThread:3953242 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 2001280 None
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research-ultimate-checking/bhq23mh5
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_restore():2570] restore
+2026-08-19 17:18:44,576 INFO    MainThread:3953242 [wandb_run.py:_restore():2576] restore done
+2026-08-19 17:18:45,156 INFO    MainThread:3953242 [wandb_run.py:_footer_sync_info():3993] logging synced files
diff --git a/zain/Activation/wandb/run-20260819_171541-bhq23mh5/run-bhq23mh5.wandb b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/run-bhq23mh5.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..c7bbce410c602310f660099133b135041ba3ed12
--- /dev/null
+++ b/zain/Activation/wandb/run-20260819_171541-bhq23mh5/run-bhq23mh5.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:896db157ba2ca29ceb3d6b02e8ae3f8c75a1864cc416843cde980ad34c0a27a7
+size 251327