Auto upload zain 2026-08-12T19:42:34.849965

.gitattributes

@@ -42,3 +42,4 @@ zain/Activation/wandb/run-20260812_184213-sjvvelzu/run-sjvvelzu.wandb filter=lfs
 zain/Activation/wandb/run-20260812_185803-0fh9o8ec/run-0fh9o8ec.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260812_191207-w1cgnksl/run-w1cgnksl.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260812_192631-1pwkdnr5/run-1pwkdnr5.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260812_194028-upy1ssu0/run-upy1ssu0.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/out/glu-silu-waleed10-150L_run/training_log.jsonl

@@ -0,0 +1,9 @@
+{"step": 20, "epoch": 0.0013481631277384564, "timestamp": 1786563638.504643, "loss": 7.965932464599609, "grad_norm": 1.2578125, "learning_rate": 0.000475, "train/total_time_seconds": 8.07786701247096, "train/time_per_step_avg": 0.40389335062354803, "train/epoch_time_elapsed": 9.141933184117079, "train/estimated_remaining_minutes": 6.5969247268512845}
+{"step": 40, "epoch": 0.002696326255476913, "timestamp": 1786563646.996639, "loss": 6.833866882324219, "grad_norm": 0.875, "learning_rate": 0.0009750000000000001, "train/total_time_seconds": 15.590525463223457, "train/time_per_step_avg": 0.3897631365805864, "train/epoch_time_elapsed": 17.633928153663874, "train/estimated_remaining_minutes": 6.236210185289383}
+{"step": 50, "epoch": 0.003370407819346141, "timestamp": 1786563673.2012448, "eval_loss": 6.037237167358398, "eval_runtime": 21.8944, "eval_samples_per_second": 435.135, "eval_steps_per_second": 3.426, "train/total_time_seconds": 19.396588761359453, "train/time_per_step_avg": 0.3879317752271891, "train/epoch_time_elapsed": 43.83853352814913, "train/estimated_remaining_minutes": 6.1422531077638265}
+{"step": 60, "epoch": 0.004044489383215369, "timestamp": 1786563677.2953885, "loss": 6.078518295288086, "grad_norm": 0.2294921875, "learning_rate": 0.001475, "train/total_time_seconds": 23.003292303532362, "train/time_per_step_avg": 0.3833882050588727, "train/epoch_time_elapsed": 47.93267811462283, "train/estimated_remaining_minutes": 6.006415212589006}
+{"step": 80, "epoch": 0.005392652510953826, "timestamp": 1786563685.5203106, "loss": 6.0116935729980465, "grad_norm": 0.119140625, "learning_rate": 0.001975, "train/total_time_seconds": 30.244462601840496, "train/time_per_step_avg": 0.3780557825230062, "train/epoch_time_elapsed": 56.157600689679384, "train/estimated_remaining_minutes": 5.796855332019429}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786563693.8344162, "loss": 5.88342514038086, "grad_norm": 0.359375, "learning_rate": 0.002475, "train/total_time_seconds": 37.563432570546865, "train/time_per_step_avg": 0.3756343257054687, "train/epoch_time_elapsed": 64.47170653939247, "train/estimated_remaining_minutes": 5.63451488558203}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786563715.4188232, "eval_loss": 5.759780406951904, "eval_runtime": 21.5827, "eval_samples_per_second": 441.419, "eval_steps_per_second": 3.475, "train/total_time_seconds": 37.563432570546865, "train/time_per_step_avg": 0.3756343257054687, "train/epoch_time_elapsed": 86.05611123517156, "train/estimated_remaining_minutes": 5.63451488558203}
+{"step": 120, "epoch": 0.008088978766430738, "timestamp": 1786563723.8760347, "loss": 5.775747299194336, "grad_norm": 0.2490234375, "learning_rate": 0.0029749999999999998, "train/total_time_seconds": 45.00043707713485, "train/time_per_step_avg": 0.36922570064663884, "train/epoch_time_elapsed": 94.51332535594702, "train/estimated_remaining_minutes": 5.500053420538704}
+{"step": 140, "epoch": 0.009437141894169195, "timestamp": 1786563732.4270985, "loss": 5.717639923095703, "grad_norm": 0.1845703125, "learning_rate": 0.003475, "train/total_time_seconds": 52.570311672985554, "train/time_per_step_avg": 0.36979786209762094, "train/epoch_time_elapsed": 103.06438939645886, "train/estimated_remaining_minutes": 5.382198576043759}

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818fa82e180602854de237ae5ae380b708c78e652c76cf1f31d0c94903188ae3
+size 50410784

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fae71c6d64b1e5d2539aa9ea6c0c5f9ef7b844853cb3f77ff31b933eb3d139b
+size 101580072

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+size 14244

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db76f238494a413266398860b344c6c4c0f3a045ef78b23af1afac6775a12d1
+size 1064

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,544 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.06740815638692282,
+  "eval_steps": 50,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.3046875,
+      "learning_rate": 0.000475,
+      "loss": 7.9166419982910154,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 0.98828125,
+      "learning_rate": 0.0009750000000000001,
+      "loss": 6.784420776367187,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.158936977386475,
+      "eval_runtime": 20.8926,
+      "eval_samples_per_second": 455.998,
+      "eval_steps_per_second": 3.59,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.2734375,
+      "learning_rate": 0.001475,
+      "loss": 6.296163558959961,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.65625,
+      "learning_rate": 0.001975,
+      "loss": 6.782315063476562,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.002475,
+      "loss": 6.999111175537109,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 6.028029441833496,
+      "eval_runtime": 20.8663,
+      "eval_samples_per_second": 456.574,
+      "eval_steps_per_second": 3.594,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.138671875,
+      "learning_rate": 0.0029749999999999998,
+      "loss": 5.9767295837402346,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.10009765625,
+      "learning_rate": 0.003475,
+      "loss": 5.927040863037109,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 5.9150004386901855,
+      "eval_runtime": 21.0395,
+      "eval_samples_per_second": 452.816,
+      "eval_steps_per_second": 3.565,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 0.003975,
+      "loss": 5.912818527221679,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 0.004475,
+      "loss": 5.916665649414062,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.10595703125,
+      "learning_rate": 0.004975,
+      "loss": 5.929853057861328,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.944804668426514,
+      "eval_runtime": 20.8873,
+      "eval_samples_per_second": 456.114,
+      "eval_steps_per_second": 3.591,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.107421875,
+      "learning_rate": 0.005,
+      "loss": 5.925020599365235,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.119140625,
+      "learning_rate": 0.005,
+      "loss": 5.879428100585938,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.852781295776367,
+      "eval_runtime": 20.8229,
+      "eval_samples_per_second": 457.525,
+      "eval_steps_per_second": 3.602,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.005,
+      "loss": 5.855681991577148,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.005,
+      "loss": 5.843220138549805,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.265625,
+      "learning_rate": 0.005,
+      "loss": 5.807944869995117,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 5.846872806549072,
+      "eval_runtime": 20.8625,
+      "eval_samples_per_second": 456.657,
+      "eval_steps_per_second": 3.595,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.671875,
+      "learning_rate": 0.005,
+      "loss": 5.938493728637695,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 0.005,
+      "loss": 5.875822067260742,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 5.84057092666626,
+      "eval_runtime": 20.8768,
+      "eval_samples_per_second": 456.345,
+      "eval_steps_per_second": 3.593,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.6640625,
+      "learning_rate": 0.005,
+      "loss": 5.8434196472167965,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.005,
+      "loss": 5.858510589599609,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.10498046875,
+      "learning_rate": 0.005,
+      "loss": 5.849637603759765,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 5.826656341552734,
+      "eval_runtime": 20.8229,
+      "eval_samples_per_second": 457.524,
+      "eval_steps_per_second": 3.602,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.130859375,
+      "learning_rate": 0.005,
+      "loss": 5.8005931854248045,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 1.8515625,
+      "learning_rate": 0.005,
+      "loss": 5.846059417724609,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 5.830883502960205,
+      "eval_runtime": 20.9783,
+      "eval_samples_per_second": 454.136,
+      "eval_steps_per_second": 3.575,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.005,
+      "loss": 5.8179374694824215,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 0.005,
+      "loss": 5.807895278930664,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.408203125,
+      "learning_rate": 0.005,
+      "loss": 5.799447631835937,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 5.79749059677124,
+      "eval_runtime": 20.9038,
+      "eval_samples_per_second": 455.755,
+      "eval_steps_per_second": 3.588,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 13.875,
+      "learning_rate": 0.005,
+      "loss": 5.859172821044922,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 1.7109375,
+      "learning_rate": 0.005,
+      "loss": 5.9967388153076175,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 5.993953227996826,
+      "eval_runtime": 20.9458,
+      "eval_samples_per_second": 454.84,
+      "eval_steps_per_second": 3.581,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 1.0625,
+      "learning_rate": 0.005,
+      "loss": 5.992317199707031,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 51.25,
+      "learning_rate": 0.005,
+      "loss": 7.135798645019531,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 6432.0,
+      "learning_rate": 0.005,
+      "loss": 8.179801177978515,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 7.734091281890869,
+      "eval_runtime": 20.8306,
+      "eval_samples_per_second": 457.357,
+      "eval_steps_per_second": 3.6,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 2784.0,
+      "learning_rate": 0.005,
+      "loss": 7.538851928710938,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 13824.0,
+      "learning_rate": 0.005,
+      "loss": 7.471035766601562,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 7.50764274597168,
+      "eval_runtime": 20.9053,
+      "eval_samples_per_second": 455.723,
+      "eval_steps_per_second": 3.588,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 334.0,
+      "learning_rate": 0.005,
+      "loss": 7.518445587158203,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 466.0,
+      "learning_rate": 0.005,
+      "loss": 7.463330078125,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 16.5,
+      "learning_rate": 0.005,
+      "loss": 7.236065673828125,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 7.128774642944336,
+      "eval_runtime": 20.8628,
+      "eval_samples_per_second": 456.651,
+      "eval_steps_per_second": 3.595,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 36.25,
+      "learning_rate": 0.005,
+      "loss": 7.062549591064453,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 8.6875,
+      "learning_rate": 0.005,
+      "loss": 7.019995880126953,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 6.980681419372559,
+      "eval_runtime": 20.8068,
+      "eval_samples_per_second": 457.879,
+      "eval_steps_per_second": 3.605,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 9.0625,
+      "learning_rate": 0.005,
+      "loss": 6.996089172363281,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 20.375,
+      "learning_rate": 0.005,
+      "loss": 6.979440307617187,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 3.71875,
+      "learning_rate": 0.005,
+      "loss": 6.933939361572266,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 6.91081428527832,
+      "eval_runtime": 20.8336,
+      "eval_samples_per_second": 457.289,
+      "eval_steps_per_second": 3.6,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 24.125,
+      "learning_rate": 0.005,
+      "loss": 6.906638336181641,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 4.84375,
+      "learning_rate": 0.005,
+      "loss": 6.94384536743164,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 6.953524589538574,
+      "eval_runtime": 20.9024,
+      "eval_samples_per_second": 455.784,
+      "eval_steps_per_second": 3.588,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 10.75,
+      "learning_rate": 0.005,
+      "loss": 6.885418701171875,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 11.75,
+      "learning_rate": 0.005,
+      "loss": 6.8612525939941404,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 63.0,
+      "learning_rate": 0.005,
+      "loss": 6.847682189941406,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 6.833151817321777,
+      "eval_runtime": 20.9196,
+      "eval_samples_per_second": 455.41,
+      "eval_steps_per_second": 3.585,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 16.125,
+      "learning_rate": 0.005,
+      "loss": 6.854830169677735,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 3.1875,
+      "learning_rate": 0.005,
+      "loss": 6.8544471740722654,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 6.827374458312988,
+      "eval_runtime": 20.9371,
+      "eval_samples_per_second": 455.03,
+      "eval_steps_per_second": 3.582,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 7.96875,
+      "learning_rate": 0.005,
+      "loss": 6.813847351074219,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 9.4375,
+      "learning_rate": 0.005,
+      "loss": 6.787914276123047,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 2.734375,
+      "learning_rate": 0.005,
+      "loss": 6.793418121337891,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 6.780795097351074,
+      "eval_runtime": 20.9425,
+      "eval_samples_per_second": 454.913,
+      "eval_steps_per_second": 3.581,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4839413121024000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/mlp-waleed10-150L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b0b0748b0191a69b4408647782104c1fcf9689dcbfd590a6d376739ba29db16
+size 4920

zain/Activation/out/mlp-waleed10-150L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-waleed10-150L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:818fa82e180602854de237ae5ae380b708c78e652c76cf1f31d0c94903188ae3
+size 50410784

zain/Activation/out/mlp-waleed10-150L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-waleed10-150L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b0b0748b0191a69b4408647782104c1fcf9689dcbfd590a6d376739ba29db16
+size 4920

zain/Activation/out/mlp-waleed10-150L_run/training_log.jsonl

@@ -55,3 +55,18 @@
 {"step": 800, "epoch": 0.053926525109538256, "timestamp": 1786563421.7254708, "loss": 6.933939361572266, "grad_norm": 3.71875, "learning_rate": 0.005, "train/total_time_seconds": 275.4879702143371, "train/time_per_step_avg": 0.34374881215393543, "train/epoch_time_elapsed": 629.0968559496105, "train/estimated_remaining_minutes": 1.1478665425597379}
 {"step": 800, "epoch": 0.053926525109538256, "timestamp": 1786563442.56066, "eval_loss": 6.91081428527832, "eval_runtime": 20.8336, "eval_samples_per_second": 457.289, "eval_steps_per_second": 3.6, "train/total_time_seconds": 275.4879702143371, "train/time_per_step_avg": 0.34374881215393543, "train/epoch_time_elapsed": 649.9320441633463, "train/estimated_remaining_minutes": 1.1478665425597379}
 {"step": 820, "epoch": 0.05527468823727671, "timestamp": 1786563450.442789, "loss": 6.906638336181641, "grad_norm": 24.125, "learning_rate": 0.005, "train/total_time_seconds": 282.3668001666665, "train/time_per_step_avg": 0.3452243387699127, "train/epoch_time_elapsed": 657.8141742534935, "train/estimated_remaining_minutes": 1.0330492689024384}
+{"step": 840, "epoch": 0.056622851365015166, "timestamp": 1786563458.321524, "loss": 6.94384536743164, "grad_norm": 4.84375, "learning_rate": 0.005, "train/total_time_seconds": 289.2485414519906, "train/time_per_step_avg": 0.3432664616033435, "train/epoch_time_elapsed": 665.6929088495672, "train/estimated_remaining_minutes": 0.9182493379428274}
+{"step": 850, "epoch": 0.057296932928884395, "timestamp": 1786563483.1368167, "eval_loss": 6.953524589538574, "eval_runtime": 20.9024, "eval_samples_per_second": 455.784, "eval_steps_per_second": 3.588, "train/total_time_seconds": 292.6652095094323, "train/time_per_step_avg": 0.3438457490503788, "train/epoch_time_elapsed": 690.5082006491721, "train/estimated_remaining_minutes": 0.8607800279689185}
+{"step": 860, "epoch": 0.057971014492753624, "timestamp": 1786563487.0316963, "loss": 6.885418701171875, "grad_norm": 10.75, "learning_rate": 0.005, "train/total_time_seconds": 296.06871227920055, "train/time_per_step_avg": 0.3432602442055941, "train/epoch_time_elapsed": 694.4030815921724, "train/estimated_remaining_minutes": 0.8032872038582961}
+{"step": 880, "epoch": 0.05931917762049208, "timestamp": 1786563494.9456503, "loss": 6.8612525939941404, "grad_norm": 11.75, "learning_rate": 0.005, "train/total_time_seconds": 302.9619455039501, "train/time_per_step_avg": 0.34365666687488555, "train/epoch_time_elapsed": 702.3170354515314, "train/estimated_remaining_minutes": 0.6885498761453411}
+{"step": 900, "epoch": 0.06066734074823053, "timestamp": 1786563502.9127316, "loss": 6.847682189941406, "grad_norm": 63.0, "learning_rate": 0.005, "train/total_time_seconds": 309.91800881177187, "train/time_per_step_avg": 0.3443003859743476, "train/epoch_time_elapsed": 710.284116987139, "train/estimated_remaining_minutes": 0.5739222385403182}
+{"step": 900, "epoch": 0.06066734074823053, "timestamp": 1786563523.834069, "eval_loss": 6.833151817321777, "eval_runtime": 20.9196, "eval_samples_per_second": 455.41, "eval_steps_per_second": 3.585, "train/total_time_seconds": 309.91800881177187, "train/time_per_step_avg": 0.3443003859743476, "train/epoch_time_elapsed": 731.2054530121386, "train/estimated_remaining_minutes": 0.5739222385403182}
+{"step": 920, "epoch": 0.06201550387596899, "timestamp": 1786563531.9341893, "loss": 6.854830169677735, "grad_norm": 16.125, "learning_rate": 0.005, "train/total_time_seconds": 317.03124503418803, "train/time_per_step_avg": 0.34664444867521527, "train/epoch_time_elapsed": 739.3055738471448, "train/estimated_remaining_minutes": 0.45946557251331593}
+{"step": 940, "epoch": 0.06336366700370745, "timestamp": 1786563539.9973881, "loss": 6.8544471740722654, "grad_norm": 3.1875, "learning_rate": 0.005, "train/total_time_seconds": 324.0328467004001, "train/time_per_step_avg": 0.3478430524840951, "train/epoch_time_elapsed": 747.3687726780772, "train/estimated_remaining_minutes": 0.3447157943621278}
+{"step": 950, "epoch": 0.06403774856757667, "timestamp": 1786563564.8508148, "eval_loss": 6.827374458312988, "eval_runtime": 20.9371, "eval_samples_per_second": 455.03, "eval_steps_per_second": 3.582, "train/total_time_seconds": 327.45656882598996, "train/time_per_step_avg": 0.34791359316557646, "train/epoch_time_elapsed": 772.222199305892, "train/estimated_remaining_minutes": 0.28724260423332454}
+{"step": 960, "epoch": 0.06471183013144591, "timestamp": 1786563568.7792199, "loss": 6.813847351074219, "grad_norm": 7.96875, "learning_rate": 0.005, "train/total_time_seconds": 330.88585337996483, "train/time_per_step_avg": 0.34817141100764276, "train/epoch_time_elapsed": 776.1506049074233, "train/estimated_remaining_minutes": 0.22978184262497559}
+{"step": 980, "epoch": 0.06605999325918437, "timestamp": 1786563576.6184173, "loss": 6.787914276123047, "grad_norm": 9.4375, "learning_rate": 0.005, "train/total_time_seconds": 337.7470153644681, "train/time_per_step_avg": 0.3478506986051798, "train/epoch_time_elapsed": 783.9898024499416, "train/estimated_remaining_minutes": 0.11487993719879867}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786563584.4612634, "loss": 6.793418121337891, "grad_norm": 2.734375, "learning_rate": 0.005, "train/total_time_seconds": 344.6105516180396, "train/time_per_step_avg": 0.3469254280626774, "train/epoch_time_elapsed": 791.8326484896243, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786563605.4052436, "eval_loss": 6.780795097351074, "eval_runtime": 20.9425, "eval_samples_per_second": 454.913, "eval_steps_per_second": 3.581, "train/total_time_seconds": 344.6105516180396, "train/time_per_step_avg": 0.3469254280626774, "train/epoch_time_elapsed": 812.7766273245215, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786563605.817271, "train_runtime": 813.897, "train_samples_per_second": 78.634, "train_steps_per_second": 1.229, "total_flos": 4839413121024000.0, "train_loss": 6.52247477722168, "train/total_time_seconds": 344.6105516180396, "train/time_per_step_avg": 0.3469254280626774, "train/epoch_time_elapsed": 813.1886536031961, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786563626.7643843, "eval_loss": 6.780795097351074, "eval_runtime": 20.9438, "eval_samples_per_second": 454.884, "eval_steps_per_second": 3.581, "train/total_time_seconds": 344.6105516180396, "train/time_per_step_avg": 0.3469254280626774, "train/epoch_time_elapsed": 834.1357678249478, "train/estimated_remaining_minutes": 0.0}

zain/Activation/wandb/debug-internal.log

@@ -1,97 +1,25 @@
-{"time":"2026-08-12T19:26:32.010129578Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-12T19:26:32.010264802Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-12T19:26:32.273779818Z","level":"INFO","msg":"stream: created new stream","id":"1pwkdnr5"}
-{"time":"2026-08-12T19:26:32.273844939Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-12T19:26:32.273961668Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-12T19:26:32.273969985Z","level":"INFO","msg":"writer: started","stream_id":"1pwkdnr5"}
-{"time":"2026-08-12T19:26:32.273991847Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-12T19:26:33.472626381Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:26:33.580547558Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:26:48.47323809Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
-{"time":"2026-08-12T19:26:48.600344001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:27:03.473267558Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:27:03.610205342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:27:18.473731434Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:27:18.592911793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:27:33.472867018Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":6,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:27:33.617645879Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:27:48.4733333Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:27:48.627027094Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:28:03.473365161Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":2,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:28:03.610068662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:28:18.473347677Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:28:18.598406722Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:28:33.472941508Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":2,"console_lines":9}
-{"time":"2026-08-12T19:28:33.587751733Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:28:48.473617719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":3,"events_offset":16,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:28:48.62236526Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:29:03.472811969Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:29:03.601804908Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:29:18.473500858Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:29:18.594013664Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:29:33.472985453Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":2,"events_offset":22,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:29:33.581058226Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:29:48.473201938Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:29:48.602549501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:30:03.473653683Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":26,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:30:03.62733505Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:30:18.472977158Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":28,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:30:18.631910458Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:30:33.472997997Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-12T19:30:33.603086056Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:30:48.473258396Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":32,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:30:48.58700914Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:31:03.473138554Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":34,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:31:03.604359922Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:31:18.473452645Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:31:18.59204066Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:31:33.473738629Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":2,"events_offset":38,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:31:33.597716205Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:31:48.473071736Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:31:48.599052698Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:32:03.472976877Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:32:03.623256777Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:32:18.47306594Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":2,"events_offset":44,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:32:18.583411294Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:32:33.473125855Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-12T19:32:33.608523173Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:32:48.473272955Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":0,"console_lines":1}
-{"time":"2026-08-12T19:32:48.610196134Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:33:03.473334604Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":2,"events_offset":50,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-12T19:33:03.601258431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:33:18.473328452Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":52,"events_lines":2,"console_offset":11,"console_lines":25}
-{"time":"2026-08-12T19:33:18.623982893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:33:33.473340917Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":2,"events_offset":54,"events_lines":2,"console_offset":36,"console_lines":2}
-{"time":"2026-08-12T19:33:33.608650042Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:33:48.473832492Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":36,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":38,"console_lines":1}
-{"time":"2026-08-12T19:33:48.599536445Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:34:03.473017641Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":39,"console_lines":1}
-{"time":"2026-08-12T19:34:03.635359326Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:34:18.472894091Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":2,"events_offset":60,"events_lines":2,"console_offset":40,"console_lines":2}
-{"time":"2026-08-12T19:34:18.600570016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:34:33.473291385Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":42,"console_lines":1}
-{"time":"2026-08-12T19:34:33.581944505Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:34:48.473584758Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":2,"events_offset":64,"events_lines":2,"console_offset":43,"console_lines":2}
-{"time":"2026-08-12T19:34:48.606668658Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:35:03.473319009Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":45,"console_lines":1}
-{"time":"2026-08-12T19:35:03.603752428Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:35:18.473487896Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
-{"time":"2026-08-12T19:35:18.602721203Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:35:33.473474753Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":44,"history_lines":3,"events_offset":70,"events_lines":2,"console_offset":46,"console_lines":3}
-{"time":"2026-08-12T19:35:33.595665264Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:35:48.472792776Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":49,"console_lines":1}
-{"time":"2026-08-12T19:35:48.616660495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:36:03.473257584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":50,"console_lines":1}
-{"time":"2026-08-12T19:36:03.658579944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:36:18.472783339Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":2,"events_offset":76,"events_lines":2,"console_offset":51,"console_lines":2}
-{"time":"2026-08-12T19:36:18.609119147Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:36:33.473066092Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
-{"time":"2026-08-12T19:36:33.578695174Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:36:48.473668777Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":51,"history_lines":2,"events_offset":80,"events_lines":2,"console_offset":53,"console_lines":2}
-{"time":"2026-08-12T19:36:48.581270138Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:37:03.473162179Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":2,"events_offset":82,"events_lines":2,"console_offset":55,"console_lines":2}
-{"time":"2026-08-12T19:37:03.605391989Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:37:18.473664851Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":84,"events_lines":2}
-{"time":"2026-08-12T19:37:18.59366684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-12T19:37:33.473166616Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":2,"events_offset":86,"events_lines":2,"console_offset":57,"console_lines":2}
-{"time":"2026-08-12T19:37:33.605827068Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:28.792818658Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T19:40:28.792960001Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T19:40:29.057378128Z","level":"INFO","msg":"stream: created new stream","id":"upy1ssu0"}
+{"time":"2026-08-12T19:40:29.057463928Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T19:40:29.057546427Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T19:40:29.057557067Z","level":"INFO","msg":"writer: started","stream_id":"upy1ssu0"}
+{"time":"2026-08-12T19:40:29.057612772Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T19:40:30.462657884Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:40:30.586881351Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:45.463697522Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-12T19:40:45.585400191Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:00.463056033Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:00.608858379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:15.463035836Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:15.618634832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:30.463506926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":2,"events_offset":6,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:30.587413688Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:45.463539103Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:45.717043114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:42:00.463686762Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:42:00.654218178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:42:15.463308103Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:42:15.599573178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:42:30.46300339Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:42:30.591443919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/debug.log

@@ -1,20 +1,20 @@
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_192631-1pwkdnr5/logs/debug.log
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_192631-1pwkdnr5/logs/debug-internal.log
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug.log
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug-internal.log
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
-2026-08-12 19:26:31,997 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
-2026-08-12 19:26:32,274 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
-2026-08-12 19:26:32,275 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
-2026-08-12 19:26:32,282 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-12 19:26:32,537 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
-2026-08-12 19:26:32,619 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-12 19:26:32,619 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-12 19:26:32,619 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-12 19:26:32,619 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-12 19:26:32,620 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-12 19:26:32,621 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-waleed10-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-waleed10-150L-25.1M-20260812-192631', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-waleed10-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-12 19:26:32,628 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c635a74990>>
-2026-08-12 19:26:32,628 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 19:40:29,058 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
+2026-08-12 19:40:29,059 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
+2026-08-12 19:40:29,066 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 19:40:29,272 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 19:40:29,354 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 19:40:29,354 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu-waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-waleed10-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-waleed10-150L-25.1M-20260812-194028', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-silu-waleed10-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 19:40:29,362 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c635af3bd0>>
+2026-08-12 19:40:29,362 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None

zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_180713-utqzlg67/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_182126-vyycsg6r/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_183823-qtstmmvv/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_185803-0fh9o8ec/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_191207-w1cgnksl/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/files/config.yaml

@@ -0,0 +1,445 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            r7p1rq02aw8r3n23icm1x711rk5uv2d7:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml
+                    - --variants
+                    - glu-silu
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-linear
+                    - glu-waleed10
+                    - mlp-waleed10
+                    - glu-silu-waleed10
+                    - mlp-silu-waleed10
+                    - glu-situglu
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "716203872256"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: c53713bcba08d1849136dad91288c4b82040634e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-12T19:26:31.995922Z"
+                writerId: r7p1rq02aw8r3n23icm1x711rk5uv2d7
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: waleed10
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 50
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-mlp-waleed10-150L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.005
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 0
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: mlp
+model/num_parameters:
+    value: 25138816
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 150
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/mlp-waleed10-150L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 128
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-mlp-waleed10-150L-25.1M-20260812-192631
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/files/output.log

@@ -57,3 +57,27 @@
 {'loss': '6.934', 'grad_norm': '3.719', 'learning_rate': '0.005', 'epoch': '0.05393', 'train/total_time_seconds': '275.5', 'train/time_per_step_avg': '0.3437', 'train/epoch_time_elapsed': '629.1', 'train/estimated_remaining_minutes': '1.148'}
 {'eval_loss': '6.911', 'eval_runtime': '20.83', 'eval_samples_per_second': '457.3', 'eval_steps_per_second': '3.6', 'epoch': '0.05393', 'train/total_time_seconds': '275.5', 'train/time_per_step_avg': '0.3437', 'train/epoch_time_elapsed': '649.9', 'train/estimated_remaining_minutes': '1.148'}
 {'loss': '6.907', 'grad_norm': '24.12', 'learning_rate': '0.005', 'epoch': '0.05527', 'train/total_time_seconds': '282.4', 'train/time_per_step_avg': '0.3452', 'train/epoch_time_elapsed': '657.8', 'train/estimated_remaining_minutes': '1.033'}
+{'loss': '6.944', 'grad_norm': '4.844', 'learning_rate': '0.005', 'epoch': '0.05662', 'train/total_time_seconds': '289.2', 'train/time_per_step_avg': '0.3433', 'train/epoch_time_elapsed': '665.7', 'train/estimated_remaining_minutes': '0.9182'}
+{'eval_loss': '6.954', 'eval_runtime': '20.9', 'eval_samples_per_second': '455.8', 'eval_steps_per_second': '3.588', 'epoch': '0.0573', 'train/total_time_seconds': '292.7', 'train/time_per_step_avg': '0.3438', 'train/epoch_time_elapsed': '690.5', 'train/estimated_remaining_minutes': '0.8608'}
+{'loss': '6.885', 'grad_norm': '10.75', 'learning_rate': '0.005', 'epoch': '0.05797', 'train/total_time_seconds': '296.1', 'train/time_per_step_avg': '0.3433', 'train/epoch_time_elapsed': '694.4', 'train/estimated_remaining_minutes': '0.8033'}
+{'loss': '6.861', 'grad_norm': '11.75', 'learning_rate': '0.005', 'epoch': '0.05932', 'train/total_time_seconds': '303', 'train/time_per_step_avg': '0.3437', 'train/epoch_time_elapsed': '702.3', 'train/estimated_remaining_minutes': '0.6885'}
+{'loss': '6.848', 'grad_norm': '63', 'learning_rate': '0.005', 'epoch': '0.06067', 'train/total_time_seconds': '309.9', 'train/time_per_step_avg': '0.3443', 'train/epoch_time_elapsed': '710.3', 'train/estimated_remaining_minutes': '0.5739'}
+{'eval_loss': '6.833', 'eval_runtime': '20.92', 'eval_samples_per_second': '455.4', 'eval_steps_per_second': '3.585', 'epoch': '0.06067', 'train/total_time_seconds': '309.9', 'train/time_per_step_avg': '0.3443', 'train/epoch_time_elapsed': '731.2', 'train/estimated_remaining_minutes': '0.5739'}
+{'loss': '6.855', 'grad_norm': '16.12', 'learning_rate': '0.005', 'epoch': '0.06202', 'train/total_time_seconds': '317', 'train/time_per_step_avg': '0.3466', 'train/epoch_time_elapsed': '739.3', 'train/estimated_remaining_minutes': '0.4595'}
+{'loss': '6.854', 'grad_norm': '3.188', 'learning_rate': '0.005', 'epoch': '0.06336', 'train/total_time_seconds': '324', 'train/time_per_step_avg': '0.3478', 'train/epoch_time_elapsed': '747.4', 'train/estimated_remaining_minutes': '0.3447'}
+{'eval_loss': '6.827', 'eval_runtime': '20.94', 'eval_samples_per_second': '455', 'eval_steps_per_second': '3.582', 'epoch': '0.06404', 'train/total_time_seconds': '327.5', 'train/time_per_step_avg': '0.3479', 'train/epoch_time_elapsed': '772.2', 'train/estimated_remaining_minutes': '0.2872'}
+{'loss': '6.814', 'grad_norm': '7.969', 'learning_rate': '0.005', 'epoch': '0.06471', 'train/total_time_seconds': '330.9', 'train/time_per_step_avg': '0.3482', 'train/epoch_time_elapsed': '776.2', 'train/estimated_remaining_minutes': '0.2298'}
+{'loss': '6.788', 'grad_norm': '9.438', 'learning_rate': '0.005', 'epoch': '0.06606', 'train/total_time_seconds': '337.7', 'train/time_per_step_avg': '0.3479', 'train/epoch_time_elapsed': '784', 'train/estimated_remaining_minutes': '0.1149'}
+{'loss': '6.793', 'grad_norm': '2.734', 'learning_rate': '0.005', 'epoch': '0.06741', 'train/total_time_seconds': '344.6', 'train/time_per_step_avg': '0.3469', 'train/epoch_time_elapsed': '791.8', 'train/estimated_remaining_minutes': '0'}
+{'eval_loss': '6.781', 'eval_runtime': '20.94', 'eval_samples_per_second': '454.9', 'eval_steps_per_second': '3.581', 'epoch': '0.06741', 'train/total_time_seconds': '344.6', 'train/time_per_step_avg': '0.3469', 'train/epoch_time_elapsed': '812.8', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00,  9.16it/s]
+100%|██████████| 1000/1000 [13:33<00:00,  1.23it/s]0:00,  9.25it/s]
+{'train_runtime': '813.9', 'train_samples_per_second': '78.63', 'train_steps_per_second': '1.229', 'train_loss': '6.522', 'epoch': '0.06741', 'train/total_time_seconds': '344.6', 'train/time_per_step_avg': '0.3469', 'train/epoch_time_elapsed': '813.2', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 75/75 [00:20<00:00,  3.64it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 10.69it/s]

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/train/total_time_seconds":344.6105516180396,"total_flos":4.839413121024e+15,"train/loss":6.793418121337891,"_runtime":834,"train/grad_norm":2.734375,"eval/samples_per_second":454.884,"train/train/estimated_remaining_minutes":0,"_step":71,"train_samples_per_second":78.634,"_timestamp":1.7865636267646284e+09,"train_runtime":813.897,"train_steps_per_second":1.229,"eval/loss":6.780795097351074,"eval/runtime":20.9438,"train/learning_rate":0.005,"eval/steps_per_second":3.581,"train/train/epoch_time_elapsed":834.1357678249478,"train/global_step":1000,"train_loss":6.52247477722168,"train/epoch":0.06740815638692282,"train/train/time_per_step_avg":0.3469254280626774,"_wandb":{"runtime":834}}

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/logs/debug-core.log

@@ -60,3 +60,10 @@
 {"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
 {"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/logs/debug-internal.log

@@ -95,3 +95,33 @@
 {"time":"2026-08-12T19:37:18.59366684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-12T19:37:33.473166616Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":2,"events_offset":86,"events_lines":2,"console_offset":57,"console_lines":2}
 {"time":"2026-08-12T19:37:33.605827068Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:37:48.473740759Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":57,"history_lines":1,"events_offset":88,"events_lines":2,"console_offset":59,"console_lines":1}
+{"time":"2026-08-12T19:37:48.585052161Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:38:03.473065053Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":60,"console_lines":1}
+{"time":"2026-08-12T19:38:03.603984154Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:38:18.473028828Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":2,"events_offset":92,"events_lines":2,"console_offset":61,"console_lines":2}
+{"time":"2026-08-12T19:38:18.627332076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:38:33.473597812Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":1,"events_offset":94,"events_lines":2,"console_offset":63,"console_lines":1}
+{"time":"2026-08-12T19:38:33.600422621Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:38:48.472962025Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":62,"history_lines":1,"events_offset":96,"events_lines":2,"console_offset":64,"console_lines":1}
+{"time":"2026-08-12T19:38:48.606409688Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:39:03.473100137Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":63,"history_lines":2,"events_offset":98,"events_lines":2,"console_offset":65,"console_lines":2}
+{"time":"2026-08-12T19:39:03.601346686Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:39:18.473165418Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
+{"time":"2026-08-12T19:39:18.596082442Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:39:33.473360146Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":2,"events_offset":102,"events_lines":2,"console_offset":67,"console_lines":2}
+{"time":"2026-08-12T19:39:33.64001896Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:39:48.473586918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":67,"history_lines":2,"events_offset":104,"events_lines":2,"console_offset":69,"console_lines":2}
+{"time":"2026-08-12T19:39:48.591959285Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:03.47312686Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
+{"time":"2026-08-12T19:40:03.621192324Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:18.473749058Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":2,"events_offset":108,"events_lines":2,"console_offset":71,"console_lines":7}
+{"time":"2026-08-12T19:40:18.592949214Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:27.365311271Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-12T19:40:27.36551004Z","level":"INFO","msg":"filestream: sending request","total_files":3,"history_offset":71,"history_lines":1,"console_offset":77,"console_lines":6,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-12T19:40:27.485460718Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:27.486808775Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-12T19:40:27.489911414Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-12T19:40:27.489929129Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-12T19:40:27.490035431Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-12T19:40:27.490039822Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/logs/debug.log

@@ -18,3 +18,8 @@ config: {'_wandb': {}}
 2026-08-12 19:26:32,621 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-waleed10-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-waleed10-150L-25.1M-20260812-192631', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-waleed10-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-12 19:26:32,628 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c635a74990>>
 2026-08-12 19:26:32,628 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None
+2026-08-12 19:40:26,900 INFO    MainThread:1371664 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research2/1pwkdnr5
+2026-08-12 19:40:26,900 INFO    MainThread:1371664 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-12 19:40:26,900 INFO    MainThread:1371664 [wandb_run.py:_restore():2570] restore
+2026-08-12 19:40:26,900 INFO    MainThread:1371664 [wandb_run.py:_restore():2576] restore done
+2026-08-12 19:40:27,489 INFO    MainThread:1371664 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260812_192631-1pwkdnr5/run-1pwkdnr5.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81837cedcd699f7dd0b8ee478b74b44f659c3d43f98412ca378244672251deb6
-size 720896
+oid sha256:e4634696f2795dd3585d5fdf22fe24ff788b7f399f30fb61231d3d8a4cae7ac1
+size 919796

zain/Activation/wandb/run-20260812_194028-upy1ssu0/files/output.log

@@ -0,0 +1,11 @@
+ 15%|█▌        | 150/1000 [01:47<05:55,  2.39it/s]  
+{'loss': '7.966', 'grad_norm': '1.258', 'learning_rate': '0.000475', 'epoch': '0.001348', 'train/total_time_seconds': '8.078', 'train/time_per_step_avg': '0.4039', 'train/epoch_time_elapsed': '9.142', 'train/estimated_remaining_minutes': '6.597'}
+{'loss': '6.834', 'grad_norm': '0.875', 'learning_rate': '0.000975', 'epoch': '0.002696', 'train/total_time_seconds': '15.59', 'train/time_per_step_avg': '0.3898', 'train/epoch_time_elapsed': '17.63', 'train/estimated_remaining_minutes': '6.236'}
+ 85%|████████▌ | 64/75 [00:18<00:03,  3.33it/s]
+{'eval_loss': '6.037', 'eval_runtime': '21.89', 'eval_samples_per_second': '435.1', 'eval_steps_per_second': '3.426', 'epoch': '0.00337', 'train/total_time_seconds': '19.4', 'train/time_per_step_avg': '0.3879', 'train/epoch_time_elapsed': '43.84', 'train/estimated_remaining_minutes': '6.142'}
+{'loss': '6.079', 'grad_norm': '0.2295', 'learning_rate': '0.001475', 'epoch': '0.004044', 'train/total_time_seconds': '23', 'train/time_per_step_avg': '0.3834', 'train/epoch_time_elapsed': '47.93', 'train/estimated_remaining_minutes': '6.006'}
+{'loss': '6.012', 'grad_norm': '0.1191', 'learning_rate': '0.001975', 'epoch': '0.005393', 'train/total_time_seconds': '30.24', 'train/time_per_step_avg': '0.3781', 'train/epoch_time_elapsed': '56.16', 'train/estimated_remaining_minutes': '5.797'}
+{'loss': '5.883', 'grad_norm': '0.3594', 'learning_rate': '0.002475', 'epoch': '0.006741', 'train/total_time_seconds': '37.56', 'train/time_per_step_avg': '0.3756', 'train/epoch_time_elapsed': '64.47', 'train/estimated_remaining_minutes': '5.635'}
+{'eval_loss': '5.76', 'eval_runtime': '21.58', 'eval_samples_per_second': '441.4', 'eval_steps_per_second': '3.475', 'epoch': '0.006741', 'train/total_time_seconds': '37.56', 'train/time_per_step_avg': '0.3756', 'train/epoch_time_elapsed': '86.06', 'train/estimated_remaining_minutes': '5.635'}
+{'loss': '5.776', 'grad_norm': '0.249', 'learning_rate': '0.002975', 'epoch': '0.008089', 'train/total_time_seconds': '45', 'train/time_per_step_avg': '0.3692', 'train/epoch_time_elapsed': '94.51', 'train/estimated_remaining_minutes': '5.5'}
+{'loss': '5.718', 'grad_norm': '0.1846', 'learning_rate': '0.003475', 'epoch': '0.009437', 'train/total_time_seconds': '52.57', 'train/time_per_step_avg': '0.3698', 'train/epoch_time_elapsed': '103.1', 'train/estimated_remaining_minutes': '5.382'}

zain/Activation/wandb/run-20260812_194028-upy1ssu0/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260812_194028-upy1ssu0/files/wandb-metadata.json

@@ -0,0 +1,107 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-12T19:40:28.790421Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml",
+    "--variants",
+    "glu-silu",
+    "glu-relu",
+    "glu-gelu",
+    "glu-sigmoid",
+    "glu-linear",
+    "glu-waleed10",
+    "mlp-waleed10",
+    "glu-silu-waleed10",
+    "mlp-silu-waleed10",
+    "glu-situglu",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "c53713bcba08d1849136dad91288c4b82040634e"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "716216700928"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "mxhd06da85qwjugpktljb0m4uwlsm7aa"
+}

zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug-core.log

@@ -0,0 +1,69 @@
+{"time":"2026-08-12T17:51:29.496049185Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7lru5pey/port-1355517.txt","pid":1355517,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-12T17:51:29.496747364Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1355517}
+{"time":"2026-08-12T17:51:29.49671311Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1355517-1357782-3052194627/socket","Net":"unix"}}
+{"time":"2026-08-12T17:51:29.674347603Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-12T17:52:11.162138091Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"2(@)"}
+{"time":"2026-08-12T17:52:11.242645021Z","level":"INFO","msg":"handleInformInit: received","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:11.508233203Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:17.001520061Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.412124918Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.887487538Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.888505Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:32.889313366Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597228944Z","level":"INFO","msg":"connection: closing","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597314364Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597242804Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597322441Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-12T17:53:03.98517338Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-12T17:53:04.076747848Z","level":"INFO","msg":"handleInformInit: received","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T17:53:04.339071164Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T17:53:09.739880255Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:11.667315857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:12.178747012Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:12.180598654Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T18:07:12.181733156Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T18:07:13.412475061Z","level":"INFO","msg":"handleInformInit: received","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:07:13.679314069Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:07:19.044675117Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:24.445193258Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:25.196343989Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:25.198067009Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:21:25.199000312Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:21:26.321522195Z","level":"INFO","msg":"handleInformInit: received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:21:26.578729863Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:21:31.999599169Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.284226521Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.818456712Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}
+{"time":"2026-08-12T18:38:21.81999609Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:21.828229465Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.104854149Z","level":"INFO","msg":"handleInformInit: received","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:23.372128976Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:38:28.777282159Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}
+{"time":"2026-08-12T18:58:01.448371065Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}
+{"time":"2026-08-12T18:58:02.001524628Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"b7qhqvo21369"}
+{"time":"2026-08-12T18:58:02.003105094Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:58:02.003794074Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"qtstmmvv","id":"3(@)"}
+{"time":"2026-08-12T18:58:03.201898424Z","level":"INFO","msg":"handleInformInit: received","streamId":"0fh9o8ec","id":"3(@)"}
+{"time":"2026-08-12T18:58:03.568010316Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0fh9o8ec","id":"3(@)"}
+{"time":"2026-08-12T18:58:09.158050393Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tdwgcmrpt4zh"}
+{"time":"2026-08-12T19:12:05.373774541Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tdwgcmrpt4zh"}
+{"time":"2026-08-12T19:12:05.978597092Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tdwgcmrpt4zh"}
+{"time":"2026-08-12T19:12:05.980085967Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"0fh9o8ec","id":"3(@)"}
+{"time":"2026-08-12T19:12:05.98075482Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"0fh9o8ec","id":"3(@)"}
+{"time":"2026-08-12T19:12:07.497066272Z","level":"INFO","msg":"handleInformInit: received","streamId":"w1cgnksl","id":"3(@)"}
+{"time":"2026-08-12T19:12:07.759791901Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"w1cgnksl","id":"3(@)"}
+{"time":"2026-08-12T19:12:13.17475902Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"at6np4o5kef7"}
+{"time":"2026-08-12T19:26:30.425472974Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"at6np4o5kef7"}
+{"time":"2026-08-12T19:26:30.923201233Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"at6np4o5kef7"}
+{"time":"2026-08-12T19:26:30.924734408Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w1cgnksl","id":"3(@)"}
+{"time":"2026-08-12T19:26:30.925492222Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w1cgnksl","id":"3(@)"}
+{"time":"2026-08-12T19:26:32.010000105Z","level":"INFO","msg":"handleInformInit: received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:26:32.273967782Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:26:37.621886772Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:26.901007747Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.488259001Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"lxypz1nseyt4"}
+{"time":"2026-08-12T19:40:27.489890537Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:27.491054457Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"1pwkdnr5","id":"3(@)"}
+{"time":"2026-08-12T19:40:28.792695046Z","level":"INFO","msg":"handleInformInit: received","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:29.057553894Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"upy1ssu0","id":"3(@)"}
+{"time":"2026-08-12T19:40:34.35643463Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"cgz0y53resja"}

zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug-internal.log

@@ -0,0 +1,25 @@
+{"time":"2026-08-12T19:40:28.792818658Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T19:40:28.792960001Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T19:40:29.057378128Z","level":"INFO","msg":"stream: created new stream","id":"upy1ssu0"}
+{"time":"2026-08-12T19:40:29.057463928Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T19:40:29.057546427Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T19:40:29.057557067Z","level":"INFO","msg":"writer: started","stream_id":"upy1ssu0"}
+{"time":"2026-08-12T19:40:29.057612772Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T19:40:30.462657884Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:40:30.586881351Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:40:45.463697522Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-12T19:40:45.585400191Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:00.463056033Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:00.608858379Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:15.463035836Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:15.618634832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:30.463506926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":2,"events_offset":6,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:30.587413688Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:41:45.463539103Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:41:45.717043114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:42:00.463686762Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:42:00.654218178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:42:15.463308103Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":12,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:42:15.599573178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T19:42:30.46300339Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T19:42:30.591443919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug.log
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_194028-upy1ssu0/logs/debug-internal.log
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-12 19:40:28,791 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 19:40:29,058 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
+2026-08-12 19:40:29,059 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
+2026-08-12 19:40:29,066 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 19:40:29,272 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 19:40:29,352 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 19:40:29,354 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 19:40:29,354 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu-waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-waleed10-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-waleed10-150L-25.1M-20260812-194028', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-silu-waleed10-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 19:40:29,362 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c635af3bd0>>
+2026-08-12 19:40:29,362 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None

zain/Activation/wandb/run-20260812_194028-upy1ssu0/run-upy1ssu0.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6205916ee26cc88f0b65fc94b58b9aa54e67a41ff9f2334f1d39715466383792
+size 131072