Auto upload zain 2026-08-14T21:14:29.212142 (part 3)

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "situglu_low",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 9,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e233ee80a9271c0f7245966ae8c5e63e06504a7f3b8db3384bfc2f7ea67ca352
+size 4011496

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3fa6126a34b3f2fea102a131368757e81c357c5ec7534a8fca12cb8760141c
+size 8074746

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b66e3cc7c452b707ddac5caf0aa17618afb9bc1a0333600a22c4afb353f3165
+size 14244

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fc3ca8af69cc9d003a05a139e7997960ab393fe31ae1d898b29038573704b4e
+size 1064

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,392 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5390835579514824,
+  "eval_steps": 60000,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01078167115902965,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.001,
+      "loss": 7.168720245361328,
+      "step": 20
+    },
+    {
+      "epoch": 0.0215633423180593,
+      "grad_norm": 0.349609375,
+      "learning_rate": 0.001,
+      "loss": 6.01421890258789,
+      "step": 40
+    },
+    {
+      "epoch": 0.03234501347708895,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.001,
+      "loss": 5.568880081176758,
+      "step": 60
+    },
+    {
+      "epoch": 0.0431266846361186,
+      "grad_norm": 0.5078125,
+      "learning_rate": 0.001,
+      "loss": 5.037441253662109,
+      "step": 80
+    },
+    {
+      "epoch": 0.05390835579514825,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.001,
+      "loss": 4.574560165405273,
+      "step": 100
+    },
+    {
+      "epoch": 0.0646900269541779,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.001,
+      "loss": 4.265085601806641,
+      "step": 120
+    },
+    {
+      "epoch": 0.07547169811320754,
+      "grad_norm": 0.84765625,
+      "learning_rate": 0.001,
+      "loss": 4.019483184814453,
+      "step": 140
+    },
+    {
+      "epoch": 0.0862533692722372,
+      "grad_norm": 0.828125,
+      "learning_rate": 0.001,
+      "loss": 3.814248275756836,
+      "step": 160
+    },
+    {
+      "epoch": 0.09703504043126684,
+      "grad_norm": 1.0,
+      "learning_rate": 0.001,
+      "loss": 3.6563358306884766,
+      "step": 180
+    },
+    {
+      "epoch": 0.1078167115902965,
+      "grad_norm": 0.875,
+      "learning_rate": 0.001,
+      "loss": 3.519593048095703,
+      "step": 200
+    },
+    {
+      "epoch": 0.11859838274932614,
+      "grad_norm": 0.57421875,
+      "learning_rate": 0.001,
+      "loss": 3.401234436035156,
+      "step": 220
+    },
+    {
+      "epoch": 0.1293800539083558,
+      "grad_norm": 0.5,
+      "learning_rate": 0.001,
+      "loss": 3.3191593170166014,
+      "step": 240
+    },
+    {
+      "epoch": 0.14016172506738545,
+      "grad_norm": 0.78125,
+      "learning_rate": 0.001,
+      "loss": 3.2113693237304686,
+      "step": 260
+    },
+    {
+      "epoch": 0.1509433962264151,
+      "grad_norm": 0.6640625,
+      "learning_rate": 0.001,
+      "loss": 3.125957489013672,
+      "step": 280
+    },
+    {
+      "epoch": 0.16172506738544473,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.001,
+      "loss": 3.0545766830444334,
+      "step": 300
+    },
+    {
+      "epoch": 0.1725067385444744,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.001,
+      "loss": 2.979182815551758,
+      "step": 320
+    },
+    {
+      "epoch": 0.18328840970350405,
+      "grad_norm": 0.8828125,
+      "learning_rate": 0.001,
+      "loss": 2.92999153137207,
+      "step": 340
+    },
+    {
+      "epoch": 0.1940700808625337,
+      "grad_norm": 0.51171875,
+      "learning_rate": 0.001,
+      "loss": 2.867526626586914,
+      "step": 360
+    },
+    {
+      "epoch": 0.20485175202156333,
+      "grad_norm": 0.6328125,
+      "learning_rate": 0.001,
+      "loss": 2.8189437866210936,
+      "step": 380
+    },
+    {
+      "epoch": 0.215633423180593,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.773433303833008,
+      "step": 400
+    },
+    {
+      "epoch": 0.22641509433962265,
+      "grad_norm": 0.62890625,
+      "learning_rate": 0.001,
+      "loss": 2.7344507217407226,
+      "step": 420
+    },
+    {
+      "epoch": 0.2371967654986523,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 2.710928726196289,
+      "step": 440
+    },
+    {
+      "epoch": 0.24797843665768193,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.6710628509521483,
+      "step": 460
+    },
+    {
+      "epoch": 0.2587601078167116,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.001,
+      "loss": 2.6398643493652343,
+      "step": 480
+    },
+    {
+      "epoch": 0.2695417789757412,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.001,
+      "loss": 2.6106098175048826,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803234501347709,
+      "grad_norm": 0.84375,
+      "learning_rate": 0.001,
+      "loss": 2.587799072265625,
+      "step": 520
+    },
+    {
+      "epoch": 0.29110512129380056,
+      "grad_norm": 0.6015625,
+      "learning_rate": 0.001,
+      "loss": 2.5707231521606446,
+      "step": 540
+    },
+    {
+      "epoch": 0.3018867924528302,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.001,
+      "loss": 2.545253944396973,
+      "step": 560
+    },
+    {
+      "epoch": 0.31266846361185985,
+      "grad_norm": 0.57421875,
+      "learning_rate": 0.001,
+      "loss": 2.5261125564575195,
+      "step": 580
+    },
+    {
+      "epoch": 0.32345013477088946,
+      "grad_norm": 0.96875,
+      "learning_rate": 0.001,
+      "loss": 2.503111457824707,
+      "step": 600
+    },
+    {
+      "epoch": 0.33423180592991913,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.4910961151123048,
+      "step": 620
+    },
+    {
+      "epoch": 0.3450134770889488,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.001,
+      "loss": 2.4673994064331053,
+      "step": 640
+    },
+    {
+      "epoch": 0.3557951482479784,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.001,
+      "loss": 2.4556552886962892,
+      "step": 660
+    },
+    {
+      "epoch": 0.3665768194070081,
+      "grad_norm": 0.9609375,
+      "learning_rate": 0.001,
+      "loss": 2.438701057434082,
+      "step": 680
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 2.4236507415771484,
+      "step": 700
+    },
+    {
+      "epoch": 0.3881401617250674,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.4043102264404297,
+      "step": 720
+    },
+    {
+      "epoch": 0.39892183288409705,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.001,
+      "loss": 2.3941486358642576,
+      "step": 740
+    },
+    {
+      "epoch": 0.40970350404312667,
+      "grad_norm": 0.76171875,
+      "learning_rate": 0.001,
+      "loss": 2.3835357666015624,
+      "step": 760
+    },
+    {
+      "epoch": 0.42048517520215634,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 2.3709957122802736,
+      "step": 780
+    },
+    {
+      "epoch": 0.431266846361186,
+      "grad_norm": 0.59375,
+      "learning_rate": 0.001,
+      "loss": 2.352398490905762,
+      "step": 800
+    },
+    {
+      "epoch": 0.4420485175202156,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.001,
+      "loss": 2.3425289154052735,
+      "step": 820
+    },
+    {
+      "epoch": 0.4528301886792453,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.001,
+      "loss": 2.3336166381835937,
+      "step": 840
+    },
+    {
+      "epoch": 0.4636118598382749,
+      "grad_norm": 0.765625,
+      "learning_rate": 0.001,
+      "loss": 2.3204221725463867,
+      "step": 860
+    },
+    {
+      "epoch": 0.4743935309973046,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.001,
+      "loss": 2.3104717254638674,
+      "step": 880
+    },
+    {
+      "epoch": 0.48517520215633425,
+      "grad_norm": 0.73828125,
+      "learning_rate": 0.001,
+      "loss": 2.300767707824707,
+      "step": 900
+    },
+    {
+      "epoch": 0.49595687331536387,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.001,
+      "loss": 2.289361572265625,
+      "step": 920
+    },
+    {
+      "epoch": 0.5067385444743935,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.001,
+      "loss": 2.283263397216797,
+      "step": 940
+    },
+    {
+      "epoch": 0.5175202156334232,
+      "grad_norm": 0.546875,
+      "learning_rate": 0.001,
+      "loss": 2.28558235168457,
+      "step": 960
+    },
+    {
+      "epoch": 0.5283018867924528,
+      "grad_norm": 0.60546875,
+      "learning_rate": 0.001,
+      "loss": 2.2729494094848635,
+      "step": 980
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.001,
+      "loss": 2.262714385986328,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5390835579514824,
+      "eval_loss": 2.262706995010376,
+      "eval_runtime": 9.5223,
+      "eval_samples_per_second": 1000.494,
+      "eval_steps_per_second": 1.995,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2323107545088000.0,
+  "train_batch_size": 512,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-situglu_low-9L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f25d0bd9783b93f4f41ff419d306235c223c206b24ccc727fc5a1503e39e36ba
+size 4920