Auto upload zain 2026-08-14T23:07:46.851605 (part 6)

zain/Activation/out/glu-waleedglu_low-21L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-waleedglu_low-21L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b710bd96acfb6390e443452d6d294cbaf327cdd790a050acb04137ff8f2a9d50
+size 4920

zain/Activation/out/glu-waleedglu_low-21L_run/training_log.jsonl

@@ -0,0 +1,53 @@
+{"step": 20, "epoch": 0.010784578053383662, "timestamp": 1786747955.5507054, "loss": 28.87822570800781, "grad_norm": 4.0, "learning_rate": 0.001, "train/total_time_seconds": 8.125255100429058, "train/time_per_step_avg": 0.4062627550214529, "train/epoch_time_elapsed": 15.911321021616459, "train/estimated_remaining_minutes": 6.63562499868373}
+{"step": 40, "epoch": 0.021569156106767323, "timestamp": 1786747971.392193, "loss": 24.06096496582031, "grad_norm": 0.921875, "learning_rate": 0.001, "train/total_time_seconds": 16.25177477672696, "train/time_per_step_avg": 0.40629436941817404, "train/epoch_time_elapsed": 31.752809405326843, "train/estimated_remaining_minutes": 6.500709910690785}
+{"step": 60, "epoch": 0.032353734160150985, "timestamp": 1786747987.2228582, "loss": 22.317706298828124, "grad_norm": 1.4140625, "learning_rate": 0.001, "train/total_time_seconds": 24.36207727342844, "train/time_per_step_avg": 0.40603462122380735, "train/epoch_time_elapsed": 47.58347408473492, "train/estimated_remaining_minutes": 6.361209065839648}
+{"step": 80, "epoch": 0.043138312213534646, "timestamp": 1786748002.850897, "loss": 20.63355407714844, "grad_norm": 2.484375, "learning_rate": 0.001, "train/total_time_seconds": 32.44940710440278, "train/time_per_step_avg": 0.40561758880503473, "train/epoch_time_elapsed": 63.2115133702755, "train/estimated_remaining_minutes": 6.219469695010533}
+{"step": 100, "epoch": 0.05392289026691831, "timestamp": 1786748018.7311933, "loss": 18.89258575439453, "grad_norm": 1.8515625, "learning_rate": 0.001, "train/total_time_seconds": 40.5365272462368, "train/time_per_step_avg": 0.40536527246236803, "train/epoch_time_elapsed": 79.09180996194482, "train/estimated_remaining_minutes": 6.0804790869355205}
+{"step": 120, "epoch": 0.06470746832030197, "timestamp": 1786748034.5811245, "loss": 17.514393615722657, "grad_norm": 2.140625, "learning_rate": 0.001, "train/total_time_seconds": 48.669238332659006, "train/time_per_step_avg": 0.40543983232229946, "train/epoch_time_elapsed": 94.94174122810364, "train/estimated_remaining_minutes": 5.948462462880545}
+{"step": 140, "epoch": 0.07549204637368563, "timestamp": 1786748050.41628, "loss": 16.459072875976563, "grad_norm": 2.671875, "learning_rate": 0.001, "train/total_time_seconds": 56.77880089357495, "train/time_per_step_avg": 0.4052702611684799, "train/epoch_time_elapsed": 110.77689604461193, "train/estimated_remaining_minutes": 5.813067710532674}
+{"step": 160, "epoch": 0.08627662442706929, "timestamp": 1786748066.2085338, "loss": 15.619828796386718, "grad_norm": 2.34375, "learning_rate": 0.001, "train/total_time_seconds": 64.8743740580976, "train/time_per_step_avg": 0.4051229678466916, "train/epoch_time_elapsed": 126.56915009394288, "train/estimated_remaining_minutes": 5.67650773008354}
+{"step": 180, "epoch": 0.09706120248045295, "timestamp": 1786748082.0450277, "loss": 14.93470458984375, "grad_norm": 2.125, "learning_rate": 0.001, "train/total_time_seconds": 73.00178475677967, "train/time_per_step_avg": 0.4055237765237689, "train/epoch_time_elapsed": 142.4056449122727, "train/estimated_remaining_minutes": 5.5427281019036405}
+{"step": 200, "epoch": 0.10784578053383662, "timestamp": 1786748097.7236798, "loss": 14.343766784667968, "grad_norm": 3.703125, "learning_rate": 0.001, "train/total_time_seconds": 81.08889551833272, "train/time_per_step_avg": 0.4055236827209592, "train/epoch_time_elapsed": 158.0842955261469, "train/estimated_remaining_minutes": 5.405926367888848}
+{"step": 220, "epoch": 0.11863035858722028, "timestamp": 1786748113.5104978, "loss": 13.889820861816407, "grad_norm": 2.1875, "learning_rate": 0.001, "train/total_time_seconds": 89.17833111807704, "train/time_per_step_avg": 0.4050909278541803, "train/epoch_time_elapsed": 173.8711139522493, "train/estimated_remaining_minutes": 5.26962865697728}
+{"step": 240, "epoch": 0.12941493664060394, "timestamp": 1786748129.3850553, "loss": 13.443467712402343, "grad_norm": 2.9375, "learning_rate": 0.001, "train/total_time_seconds": 97.44434486329556, "train/time_per_step_avg": 0.406655439697206, "train/epoch_time_elapsed": 189.74567170068622, "train/estimated_remaining_minutes": 5.142895978896154}
+{"step": 260, "epoch": 0.1401995146939876, "timestamp": 1786748145.3215604, "loss": 12.998054504394531, "grad_norm": 1.6484375, "learning_rate": 0.001, "train/total_time_seconds": 105.56989968568087, "train/time_per_step_avg": 0.4069552562758327, "train/epoch_time_elapsed": 205.68217654898763, "train/estimated_remaining_minutes": 5.007802933807938}
+{"step": 280, "epoch": 0.15098409274737126, "timestamp": 1786748161.172112, "loss": 12.626632690429688, "grad_norm": 2.453125, "learning_rate": 0.001, "train/total_time_seconds": 113.67667726427317, "train/time_per_step_avg": 0.406748925074935, "train/epoch_time_elapsed": 221.53272734954953, "train/estimated_remaining_minutes": 4.871857597040279}
+{"step": 300, "epoch": 0.1617686708007549, "timestamp": 1786748176.9442291, "loss": 12.287302398681641, "grad_norm": 2.46875, "learning_rate": 0.001, "train/total_time_seconds": 121.78035191074014, "train/time_per_step_avg": 0.40691456392407416, "train/epoch_time_elapsed": 237.30484534427524, "train/estimated_remaining_minutes": 4.735902574306562}
+{"step": 320, "epoch": 0.17255324885413859, "timestamp": 1786748192.868352, "loss": 11.972341156005859, "grad_norm": 1.765625, "learning_rate": 0.001, "train/total_time_seconds": 129.87322784215212, "train/time_per_step_avg": 0.4069489672407508, "train/epoch_time_elapsed": 253.228967346251, "train/estimated_remaining_minutes": 4.599676819409554}
+{"step": 340, "epoch": 0.18333782690752223, "timestamp": 1786748208.6168966, "loss": 11.687677764892578, "grad_norm": 2.359375, "learning_rate": 0.001, "train/total_time_seconds": 137.99583918601274, "train/time_per_step_avg": 0.4055149432271719, "train/epoch_time_elapsed": 268.97751319408417, "train/estimated_remaining_minutes": 4.464571267782765}
+{"step": 360, "epoch": 0.1941224049609059, "timestamp": 1786748224.4517117, "loss": 11.4271728515625, "grad_norm": 2.078125, "learning_rate": 0.001, "train/total_time_seconds": 146.12659545615315, "train/time_per_step_avg": 0.4055669577047229, "train/epoch_time_elapsed": 284.812327362597, "train/estimated_remaining_minutes": 4.329676902404538}
+{"step": 380, "epoch": 0.20490698301428956, "timestamp": 1786748240.3111615, "loss": 11.163407135009766, "grad_norm": 2.0, "learning_rate": 0.001, "train/total_time_seconds": 154.2320696786046, "train/time_per_step_avg": 0.40555392414331437, "train/epoch_time_elapsed": 300.67177828401327, "train/estimated_remaining_minutes": 4.194029964944511}
+{"step": 400, "epoch": 0.21569156106767323, "timestamp": 1786748256.2138653, "loss": 10.960685729980469, "grad_norm": 1.9453125, "learning_rate": 0.001, "train/total_time_seconds": 162.36926352605224, "train/time_per_step_avg": 0.405889116153121, "train/epoch_time_elapsed": 316.57448191568255, "train/estimated_remaining_minutes": 4.059231588151306}
+{"step": 420, "epoch": 0.22647613912105688, "timestamp": 1786748272.037616, "loss": 10.743543243408203, "grad_norm": 1.9375, "learning_rate": 0.001, "train/total_time_seconds": 170.50094760209322, "train/time_per_step_avg": 0.406277197599411, "train/epoch_time_elapsed": 332.39823308214545, "train/estimated_remaining_minutes": 3.924228159095797}
+{"step": 440, "epoch": 0.23726071717444056, "timestamp": 1786748287.8980882, "loss": 10.606502532958984, "grad_norm": 2.421875, "learning_rate": 0.001, "train/total_time_seconds": 178.61680870130658, "train/time_per_step_avg": 0.40620969515293837, "train/epoch_time_elapsed": 348.2587040960789, "train/estimated_remaining_minutes": 3.788841396694382}
+{"step": 460, "epoch": 0.2480452952278242, "timestamp": 1786748303.573111, "loss": 10.407550048828124, "grad_norm": 2.375, "learning_rate": 0.001, "train/total_time_seconds": 186.72400549799204, "train/time_per_step_avg": 0.40597410041838883, "train/epoch_time_elapsed": 363.933727145195, "train/estimated_remaining_minutes": 3.6532957597433224}
+{"step": 480, "epoch": 0.2588298732812079, "timestamp": 1786748319.429599, "loss": 10.243016052246094, "grad_norm": 2.109375, "learning_rate": 0.001, "train/total_time_seconds": 194.82527919858694, "train/time_per_step_avg": 0.4059320951998234, "train/epoch_time_elapsed": 379.79021544381976, "train/estimated_remaining_minutes": 3.5176786521967087}
+{"step": 500, "epoch": 0.26961445133459155, "timestamp": 1786748335.175178, "loss": 10.094182586669922, "grad_norm": 1.953125, "learning_rate": 0.001, "train/total_time_seconds": 202.9594863653183, "train/time_per_step_avg": 0.4059022283926606, "train/epoch_time_elapsed": 395.5357943326235, "train/estimated_remaining_minutes": 3.3826581060886385}
+{"step": 520, "epoch": 0.2803990293879752, "timestamp": 1786748351.1012967, "loss": 9.965821075439454, "grad_norm": 2.28125, "learning_rate": 0.001, "train/total_time_seconds": 211.05544440820813, "train/time_per_step_avg": 0.40554496806114915, "train/epoch_time_elapsed": 411.46191192790866, "train/estimated_remaining_minutes": 3.247006837049356}
+{"step": 540, "epoch": 0.29118360744135885, "timestamp": 1786748366.8551178, "loss": 9.849958038330078, "grad_norm": 2.296875, "learning_rate": 0.001, "train/total_time_seconds": 219.1496572867036, "train/time_per_step_avg": 0.4053284858539701, "train/epoch_time_elapsed": 427.2157339602709, "train/estimated_remaining_minutes": 3.1113840232062855}
+{"step": 560, "epoch": 0.3019681854947425, "timestamp": 1786748382.6123364, "loss": 9.730626678466797, "grad_norm": 2.140625, "learning_rate": 0.001, "train/total_time_seconds": 227.23986832424998, "train/time_per_step_avg": 0.4051586282625794, "train/epoch_time_elapsed": 442.9729521200061, "train/estimated_remaining_minutes": 2.9757601804366067}
+{"step": 580, "epoch": 0.3127527635481262, "timestamp": 1786748398.3827922, "loss": 9.614877319335937, "grad_norm": 1.859375, "learning_rate": 0.001, "train/total_time_seconds": 235.33354591205716, "train/time_per_step_avg": 0.4050826671347022, "train/epoch_time_elapsed": 458.74340914189816, "train/estimated_remaining_minutes": 2.840232450662759}
+{"step": 600, "epoch": 0.3235373416015098, "timestamp": 1786748414.3937943, "loss": 9.505727386474609, "grad_norm": 2.296875, "learning_rate": 0.001, "train/total_time_seconds": 243.46157986670732, "train/time_per_step_avg": 0.40502093501389025, "train/epoch_time_elapsed": 474.7544106952846, "train/estimated_remaining_minutes": 2.705128665185637}
+{"step": 620, "epoch": 0.3343219196548935, "timestamp": 1786748430.427414, "loss": 9.414925384521485, "grad_norm": 2.046875, "learning_rate": 0.001, "train/total_time_seconds": 251.58637992665172, "train/time_per_step_avg": 0.40530935518443584, "train/epoch_time_elapsed": 490.7880291491747, "train/estimated_remaining_minutes": 2.569968397100206}
+{"step": 640, "epoch": 0.34510649770827717, "timestamp": 1786748446.4542549, "loss": 9.328840637207032, "grad_norm": 1.9921875, "learning_rate": 0.001, "train/total_time_seconds": 259.7155615314841, "train/time_per_step_avg": 0.40565904244780543, "train/epoch_time_elapsed": 506.8148699365556, "train/estimated_remaining_minutes": 2.4348333893576637}
+{"step": 660, "epoch": 0.35589107576166085, "timestamp": 1786748462.320152, "loss": 9.245575714111329, "grad_norm": 1.9609375, "learning_rate": 0.001, "train/total_time_seconds": 267.84740838781, "train/time_per_step_avg": 0.4060754006356001, "train/epoch_time_elapsed": 522.68076909706, "train/estimated_remaining_minutes": 2.2996999710064494}
+{"step": 680, "epoch": 0.36667565381504447, "timestamp": 1786748478.2107358, "loss": 9.160669708251953, "grad_norm": 2.078125, "learning_rate": 0.001, "train/total_time_seconds": 275.9440933018923, "train/time_per_step_avg": 0.40610547389835117, "train/epoch_time_elapsed": 538.5713514313102, "train/estimated_remaining_minutes": 2.164267398446214}
+{"step": 700, "epoch": 0.37746023186842814, "timestamp": 1786748493.9718537, "loss": 9.08907699584961, "grad_norm": 1.59375, "learning_rate": 0.001, "train/total_time_seconds": 284.0659934170544, "train/time_per_step_avg": 0.4060441355034709, "train/epoch_time_elapsed": 554.332470394671, "train/estimated_remaining_minutes": 2.029042810121817}
+{"step": 720, "epoch": 0.3882448099218118, "timestamp": 1786748510.2042859, "loss": 8.998544311523437, "grad_norm": 1.921875, "learning_rate": 0.001, "train/total_time_seconds": 292.20687505602837, "train/time_per_step_avg": 0.4062049512937665, "train/epoch_time_elapsed": 570.5649027526379, "train/estimated_remaining_minutes": 1.8939334494372209}
+{"step": 740, "epoch": 0.3990293879751955, "timestamp": 1786748526.0544066, "loss": 8.961595153808593, "grad_norm": 1.8515625, "learning_rate": 0.001, "train/total_time_seconds": 300.34223844110966, "train/time_per_step_avg": 0.4062667690962553, "train/epoch_time_elapsed": 586.4150231704116, "train/estimated_remaining_minutes": 1.7587608557362278}
+{"step": 760, "epoch": 0.4098139660285791, "timestamp": 1786748541.9264393, "loss": 8.89373779296875, "grad_norm": 1.78125, "learning_rate": 0.001, "train/total_time_seconds": 308.4568819515407, "train/time_per_step_avg": 0.4060947356373072, "train/epoch_time_elapsed": 602.2870550639927, "train/estimated_remaining_minutes": 1.6234572734291617}
+{"step": 780, "epoch": 0.4205985440819628, "timestamp": 1786748557.5973103, "loss": 8.858226776123047, "grad_norm": 1.90625, "learning_rate": 0.001, "train/total_time_seconds": 316.5688179396093, "train/time_per_step_avg": 0.4062472463771701, "train/epoch_time_elapsed": 617.9579257294536, "train/estimated_remaining_minutes": 1.4881440159554282}
+{"step": 800, "epoch": 0.43138312213534646, "timestamp": 1786748573.3539927, "loss": 8.755474090576172, "grad_norm": 1.8359375, "learning_rate": 0.001, "train/total_time_seconds": 324.6625647544861, "train/time_per_step_avg": 0.4059657133743167, "train/epoch_time_elapsed": 633.7146084494889, "train/estimated_remaining_minutes": 1.3527606864770254}
+{"step": 820, "epoch": 0.44216770018873014, "timestamp": 1786748589.0057478, "loss": 8.732923126220703, "grad_norm": 1.796875, "learning_rate": 0.001, "train/total_time_seconds": 332.75162014365196, "train/time_per_step_avg": 0.40544745087623596, "train/epoch_time_elapsed": 649.3663630895317, "train/estimated_remaining_minutes": 1.217383976135312}
+{"step": 840, "epoch": 0.45295227824211376, "timestamp": 1786748604.9225793, "loss": 8.691609954833984, "grad_norm": 1.8203125, "learning_rate": 0.001, "train/total_time_seconds": 340.86094953864813, "train/time_per_step_avg": 0.40518711097538473, "train/epoch_time_elapsed": 665.2831956408918, "train/estimated_remaining_minutes": 1.0820982525036449}
+{"step": 860, "epoch": 0.46373685629549743, "timestamp": 1786748620.8106012, "loss": 8.623181915283203, "grad_norm": 1.921875, "learning_rate": 0.001, "train/total_time_seconds": 348.9605983234942, "train/time_per_step_avg": 0.40503716371953485, "train/epoch_time_elapsed": 681.1712169833481, "train/estimated_remaining_minutes": 0.9467923210327363}
+{"step": 880, "epoch": 0.4745214343488811, "timestamp": 1786748636.4933302, "loss": 8.585681915283203, "grad_norm": 1.84375, "learning_rate": 0.001, "train/total_time_seconds": 357.0373779311776, "train/time_per_step_avg": 0.40468559991568326, "train/epoch_time_elapsed": 696.8539464510977, "train/estimated_remaining_minutes": 0.8114485862072218}
+{"step": 900, "epoch": 0.4853060124022648, "timestamp": 1786748652.084348, "loss": 8.534693145751953, "grad_norm": 1.9453125, "learning_rate": 0.001, "train/total_time_seconds": 365.11572790145874, "train/time_per_step_avg": 0.40453163146972654, "train/epoch_time_elapsed": 712.4449638240039, "train/estimated_remaining_minutes": 0.6761402368545533}
+{"step": 920, "epoch": 0.4960905904556484, "timestamp": 1786748668.1569273, "loss": 8.493429565429688, "grad_norm": 1.875, "learning_rate": 0.001, "train/total_time_seconds": 373.26814346015453, "train/time_per_step_avg": 0.40516523316502573, "train/epoch_time_elapsed": 728.5175438411534, "train/estimated_remaining_minutes": 0.5409683238552964}
+{"step": 940, "epoch": 0.5068751685090321, "timestamp": 1786748684.0965688, "loss": 8.474921417236327, "grad_norm": 1.640625, "learning_rate": 0.001, "train/total_time_seconds": 381.41837418451905, "train/time_per_step_avg": 0.40557424645870926, "train/epoch_time_elapsed": 744.4571851566434, "train/estimated_remaining_minutes": 0.40576422785587135}
+{"step": 960, "epoch": 0.5176597465624158, "timestamp": 1786748699.8346756, "loss": 8.457220458984375, "grad_norm": 1.7890625, "learning_rate": 0.001, "train/total_time_seconds": 389.5119272880256, "train/time_per_step_avg": 0.4055132896453142, "train/epoch_time_elapsed": 760.1952919065952, "train/estimated_remaining_minutes": 0.2704943939500178}
+{"step": 980, "epoch": 0.5284443246157994, "timestamp": 1786748715.6499183, "loss": 8.418341064453125, "grad_norm": 1.6953125, "learning_rate": 0.001, "train/total_time_seconds": 397.64077776297927, "train/time_per_step_avg": 0.4060339983180165, "train/epoch_time_elapsed": 776.0105344839394, "train/estimated_remaining_minutes": 0.13525196522550315}
+{"step": 1000, "epoch": 0.5392289026691831, "timestamp": 1786748731.611965, "loss": 8.385391235351562, "grad_norm": 1.703125, "learning_rate": 0.001, "train/total_time_seconds": 405.79406728595495, "train/time_per_step_avg": 0.4067833938449621, "train/epoch_time_elapsed": 791.9725815802813, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.5392289026691831, "timestamp": 1786748741.299589, "eval_loss": 2.096169948577881, "eval_runtime": 9.686, "eval_samples_per_second": 983.588, "eval_steps_per_second": 7.743, "train/total_time_seconds": 405.79406728595495, "train/time_per_step_avg": 0.4067833938449621, "train/epoch_time_elapsed": 801.6602005288005, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.5392289026691831, "timestamp": 1786748741.3706875, "train_runtime": 802.5252, "train_samples_per_second": 637.986, "train_steps_per_second": 1.246, "total_flos": 5420315836416000.0, "train_loss": 11.859544631958007, "train/total_time_seconds": 405.79406728595495, "train/time_per_step_avg": 0.4067833938449621, "train/epoch_time_elapsed": 801.731302190572, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.5392289026691831, "timestamp": 1786748751.1893418, "eval_loss": 2.096169948577881, "eval_runtime": 9.8163, "eval_samples_per_second": 970.533, "eval_steps_per_second": 7.64, "train/total_time_seconds": 405.79406728595495, "train/time_per_step_avg": 0.4067833938449621, "train/epoch_time_elapsed": 811.5499559640884, "train/estimated_remaining_minutes": 0.0}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9b1687dfd991ff042f68918b21a9bdfd5c2562b648e54a3cdbf2d610b3a9c9
+size 7959296

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e535ffd2007a8299d146c102ef2a448eb0bc6ac87f554927192f35c0e771b6
+size 16023738

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:906745ab5f61e73e8c1ba850ef3b839e2766eba889b49c843836266988895165
+size 1064

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/trainer_state.json

@@ -0,0 +1,69 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.05392289026691831,
+  "eval_steps": 60000,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.09375,
+      "learning_rate": 0.001,
+      "loss": 29.103460693359374,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.001,
+      "loss": 24.323658752441407,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.001,
+      "loss": 22.974208068847656,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 1.625,
+      "learning_rate": 0.001,
+      "loss": 21.970303344726563,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.625,
+      "learning_rate": 0.001,
+      "loss": 20.951182556152343,
+      "step": 100
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 542031583641600.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda593b8a69e45e9f48a5e12fceefb418c681431f4313d1ee1004dc70bcfa0dd
+size 4920

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ba794fb6fcee5b52be8662d141115e7a75521492fb734a5fb441096c5899460
+size 7959296

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a30891a43b0159714e9af5b848f983099e515a8c934604658fe059bf1e5e70b
+size 16023738

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b66e3cc7c452b707ddac5caf0aa17618afb9bc1a0333600a22c4afb353f3165
+size 14244

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fc3ca8af69cc9d003a05a139e7997960ab393fe31ae1d898b29038573704b4e
+size 1064

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,392 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5392289026691831,
+  "eval_steps": 60000,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.09375,
+      "learning_rate": 0.001,
+      "loss": 29.103460693359374,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.001,
+      "loss": 24.323658752441407,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.001,
+      "loss": 22.974208068847656,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 1.625,
+      "learning_rate": 0.001,
+      "loss": 21.970303344726563,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.625,
+      "learning_rate": 0.001,
+      "loss": 20.951182556152343,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.375,
+      "learning_rate": 0.001,
+      "loss": 20.1780517578125,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.796875,
+      "learning_rate": 0.001,
+      "loss": 19.392193603515626,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.001,
+      "loss": 18.65845489501953,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.609375,
+      "learning_rate": 0.001,
+      "loss": 17.959799194335936,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 17.317408752441406,
+      "step": 200
+    },
+    {
+      "epoch": 0.11863035858722028,
+      "grad_norm": 3.484375,
+      "learning_rate": 0.001,
+      "loss": 16.665341186523438,
+      "step": 220
+    },
+    {
+      "epoch": 0.12941493664060394,
+      "grad_norm": 4.0625,
+      "learning_rate": 0.001,
+      "loss": 16.151258850097655,
+      "step": 240
+    },
+    {
+      "epoch": 0.1401995146939876,
+      "grad_norm": 2.34375,
+      "learning_rate": 0.001,
+      "loss": 15.6710205078125,
+      "step": 260
+    },
+    {
+      "epoch": 0.15098409274737126,
+      "grad_norm": 2.984375,
+      "learning_rate": 0.001,
+      "loss": 15.264056396484374,
+      "step": 280
+    },
+    {
+      "epoch": 0.1617686708007549,
+      "grad_norm": 1.8125,
+      "learning_rate": 0.001,
+      "loss": 14.908853149414062,
+      "step": 300
+    },
+    {
+      "epoch": 0.17255324885413859,
+      "grad_norm": 2.390625,
+      "learning_rate": 0.001,
+      "loss": 14.611824035644531,
+      "step": 320
+    },
+    {
+      "epoch": 0.18333782690752223,
+      "grad_norm": 1.6484375,
+      "learning_rate": 0.001,
+      "loss": 14.270545959472656,
+      "step": 340
+    },
+    {
+      "epoch": 0.1941224049609059,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 13.935658264160157,
+      "step": 360
+    },
+    {
+      "epoch": 0.20490698301428956,
+      "grad_norm": 2.75,
+      "learning_rate": 0.001,
+      "loss": 13.626214599609375,
+      "step": 380
+    },
+    {
+      "epoch": 0.21569156106767323,
+      "grad_norm": 2.6875,
+      "learning_rate": 0.001,
+      "loss": 13.423133850097656,
+      "step": 400
+    },
+    {
+      "epoch": 0.22647613912105688,
+      "grad_norm": 2.640625,
+      "learning_rate": 0.001,
+      "loss": 13.1606201171875,
+      "step": 420
+    },
+    {
+      "epoch": 0.23726071717444056,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 12.97046356201172,
+      "step": 440
+    },
+    {
+      "epoch": 0.2480452952278242,
+      "grad_norm": 2.375,
+      "learning_rate": 0.001,
+      "loss": 12.767843627929688,
+      "step": 460
+    },
+    {
+      "epoch": 0.2588298732812079,
+      "grad_norm": 1.9609375,
+      "learning_rate": 0.001,
+      "loss": 12.555582427978516,
+      "step": 480
+    },
+    {
+      "epoch": 0.26961445133459155,
+      "grad_norm": 1.6640625,
+      "learning_rate": 0.001,
+      "loss": 12.363282012939454,
+      "step": 500
+    },
+    {
+      "epoch": 0.2803990293879752,
+      "grad_norm": 2.40625,
+      "learning_rate": 0.001,
+      "loss": 12.229927825927735,
+      "step": 520
+    },
+    {
+      "epoch": 0.29118360744135885,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.001,
+      "loss": 12.06595230102539,
+      "step": 540
+    },
+    {
+      "epoch": 0.3019681854947425,
+      "grad_norm": 2.25,
+      "learning_rate": 0.001,
+      "loss": 11.916503143310546,
+      "step": 560
+    },
+    {
+      "epoch": 0.3127527635481262,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.001,
+      "loss": 11.778754425048827,
+      "step": 580
+    },
+    {
+      "epoch": 0.3235373416015098,
+      "grad_norm": 2.5625,
+      "learning_rate": 0.001,
+      "loss": 11.674578857421874,
+      "step": 600
+    },
+    {
+      "epoch": 0.3343219196548935,
+      "grad_norm": 1.9921875,
+      "learning_rate": 0.001,
+      "loss": 11.54565658569336,
+      "step": 620
+    },
+    {
+      "epoch": 0.34510649770827717,
+      "grad_norm": 2.0,
+      "learning_rate": 0.001,
+      "loss": 11.434559631347657,
+      "step": 640
+    },
+    {
+      "epoch": 0.35589107576166085,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.001,
+      "loss": 11.319635772705078,
+      "step": 660
+    },
+    {
+      "epoch": 0.36667565381504447,
+      "grad_norm": 2.15625,
+      "learning_rate": 0.001,
+      "loss": 11.202098846435547,
+      "step": 680
+    },
+    {
+      "epoch": 0.37746023186842814,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.001,
+      "loss": 11.103318023681641,
+      "step": 700
+    },
+    {
+      "epoch": 0.3882448099218118,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 10.984959411621094,
+      "step": 720
+    },
+    {
+      "epoch": 0.3990293879751955,
+      "grad_norm": 2.78125,
+      "learning_rate": 0.001,
+      "loss": 10.932718658447266,
+      "step": 740
+    },
+    {
+      "epoch": 0.4098139660285791,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.001,
+      "loss": 10.841696166992188,
+      "step": 760
+    },
+    {
+      "epoch": 0.4205985440819628,
+      "grad_norm": 2.53125,
+      "learning_rate": 0.001,
+      "loss": 10.763890838623047,
+      "step": 780
+    },
+    {
+      "epoch": 0.43138312213534646,
+      "grad_norm": 2.375,
+      "learning_rate": 0.001,
+      "loss": 10.633779907226563,
+      "step": 800
+    },
+    {
+      "epoch": 0.44216770018873014,
+      "grad_norm": 2.46875,
+      "learning_rate": 0.001,
+      "loss": 10.587508392333984,
+      "step": 820
+    },
+    {
+      "epoch": 0.45295227824211376,
+      "grad_norm": 2.53125,
+      "learning_rate": 0.001,
+      "loss": 10.518219757080079,
+      "step": 840
+    },
+    {
+      "epoch": 0.46373685629549743,
+      "grad_norm": 2.484375,
+      "learning_rate": 0.001,
+      "loss": 10.428038787841796,
+      "step": 860
+    },
+    {
+      "epoch": 0.4745214343488811,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.001,
+      "loss": 10.359873199462891,
+      "step": 880
+    },
+    {
+      "epoch": 0.4853060124022648,
+      "grad_norm": 2.640625,
+      "learning_rate": 0.001,
+      "loss": 10.303240966796874,
+      "step": 900
+    },
+    {
+      "epoch": 0.4960905904556484,
+      "grad_norm": 2.703125,
+      "learning_rate": 0.001,
+      "loss": 10.233261108398438,
+      "step": 920
+    },
+    {
+      "epoch": 0.5068751685090321,
+      "grad_norm": 2.984375,
+      "learning_rate": 0.001,
+      "loss": 10.198040008544922,
+      "step": 940
+    },
+    {
+      "epoch": 0.5176597465624158,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.001,
+      "loss": 10.153294372558594,
+      "step": 960
+    },
+    {
+      "epoch": 0.5284443246157994,
+      "grad_norm": 2.46875,
+      "learning_rate": 0.001,
+      "loss": 10.112586212158202,
+      "step": 980
+    },
+    {
+      "epoch": 0.5392289026691831,
+      "grad_norm": 2.609375,
+      "learning_rate": 0.001,
+      "loss": 10.053606414794922,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5392289026691831,
+      "eval_loss": 2.5052456855773926,
+      "eval_runtime": 9.4135,
+      "eval_samples_per_second": 1012.06,
+      "eval_steps_per_second": 7.967,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5420315836416000.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda593b8a69e45e9f48a5e12fceefb418c681431f4313d1ee1004dc70bcfa0dd
+size 4920

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e47052a6a249ebf53373e960f46db44c16badf55d0b1f5641d7b6fe7bcd56add
+size 7959296

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a689448d286470826d3e90b92a69afbf00fcf27b42685e171aceb475c52d12bc
+size 16023738

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3b5a970fc92af42664e84ba686059eaaacfaa668691c0cf6d2afddda677461
+size 1064

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/trainer_state.json

@@ -0,0 +1,104 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.10784578053383662,
+  "eval_steps": 60000,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.09375,
+      "learning_rate": 0.001,
+      "loss": 29.103460693359374,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.001,
+      "loss": 24.323658752441407,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.001,
+      "loss": 22.974208068847656,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 1.625,
+      "learning_rate": 0.001,
+      "loss": 21.970303344726563,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.625,
+      "learning_rate": 0.001,
+      "loss": 20.951182556152343,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.375,
+      "learning_rate": 0.001,
+      "loss": 20.1780517578125,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.796875,
+      "learning_rate": 0.001,
+      "loss": 19.392193603515626,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.001,
+      "loss": 18.65845489501953,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.609375,
+      "learning_rate": 0.001,
+      "loss": 17.959799194335936,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 17.317408752441406,
+      "step": 200
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1084063167283200.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda593b8a69e45e9f48a5e12fceefb418c681431f4313d1ee1004dc70bcfa0dd
+size 4920

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f515f27d365a121e29deb1fb378e30dd334e121d4cb62d906d24656b71f2113e
+size 7959296

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:326ff18127cbac66b90e9041ed245dfd03d7ad9b8654dc096a075bfe445183a6
+size 16023738

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76cd21f32221b90fb8288fd2428595ec8b5039fe2425086d67cacb5c8d814c9e
+size 1064

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/trainer_state.json

@@ -0,0 +1,139 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1617686708007549,
+  "eval_steps": 60000,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.09375,
+      "learning_rate": 0.001,
+      "loss": 29.103460693359374,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.001,
+      "loss": 24.323658752441407,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.001,
+      "loss": 22.974208068847656,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 1.625,
+      "learning_rate": 0.001,
+      "loss": 21.970303344726563,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.625,
+      "learning_rate": 0.001,
+      "loss": 20.951182556152343,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.375,
+      "learning_rate": 0.001,
+      "loss": 20.1780517578125,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.796875,
+      "learning_rate": 0.001,
+      "loss": 19.392193603515626,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.001,
+      "loss": 18.65845489501953,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.609375,
+      "learning_rate": 0.001,
+      "loss": 17.959799194335936,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 17.317408752441406,
+      "step": 200
+    },
+    {
+      "epoch": 0.11863035858722028,
+      "grad_norm": 3.484375,
+      "learning_rate": 0.001,
+      "loss": 16.665341186523438,
+      "step": 220
+    },
+    {
+      "epoch": 0.12941493664060394,
+      "grad_norm": 4.0625,
+      "learning_rate": 0.001,
+      "loss": 16.151258850097655,
+      "step": 240
+    },
+    {
+      "epoch": 0.1401995146939876,
+      "grad_norm": 2.34375,
+      "learning_rate": 0.001,
+      "loss": 15.6710205078125,
+      "step": 260
+    },
+    {
+      "epoch": 0.15098409274737126,
+      "grad_norm": 2.984375,
+      "learning_rate": 0.001,
+      "loss": 15.264056396484374,
+      "step": 280
+    },
+    {
+      "epoch": 0.1617686708007549,
+      "grad_norm": 1.8125,
+      "learning_rate": 0.001,
+      "loss": 14.908853149414062,
+      "step": 300
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1626094750924800.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda593b8a69e45e9f48a5e12fceefb418c681431f4313d1ee1004dc70bcfa0dd
+size 4920

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a38c2c8a915b9b6f40c28c42f072a644b4cf5db90c25784335b26df0418b09b
+size 7959296

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4855eab9d35bda1625a50a9b1b82dd1490b389965177b34af7bab3d07f5e0396
+size 16023738

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51769a11c87b2b665cbe64c58b934afdb1fa1998bffb4addcc2f852b172d6681
+size 1064

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/trainer_state.json

@@ -0,0 +1,174 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.21569156106767323,
+  "eval_steps": 60000,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010784578053383662,
+      "grad_norm": 4.09375,
+      "learning_rate": 0.001,
+      "loss": 29.103460693359374,
+      "step": 20
+    },
+    {
+      "epoch": 0.021569156106767323,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.001,
+      "loss": 24.323658752441407,
+      "step": 40
+    },
+    {
+      "epoch": 0.032353734160150985,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.001,
+      "loss": 22.974208068847656,
+      "step": 60
+    },
+    {
+      "epoch": 0.043138312213534646,
+      "grad_norm": 1.625,
+      "learning_rate": 0.001,
+      "loss": 21.970303344726563,
+      "step": 80
+    },
+    {
+      "epoch": 0.05392289026691831,
+      "grad_norm": 3.625,
+      "learning_rate": 0.001,
+      "loss": 20.951182556152343,
+      "step": 100
+    },
+    {
+      "epoch": 0.06470746832030197,
+      "grad_norm": 3.375,
+      "learning_rate": 0.001,
+      "loss": 20.1780517578125,
+      "step": 120
+    },
+    {
+      "epoch": 0.07549204637368563,
+      "grad_norm": 3.796875,
+      "learning_rate": 0.001,
+      "loss": 19.392193603515626,
+      "step": 140
+    },
+    {
+      "epoch": 0.08627662442706929,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.001,
+      "loss": 18.65845489501953,
+      "step": 160
+    },
+    {
+      "epoch": 0.09706120248045295,
+      "grad_norm": 2.609375,
+      "learning_rate": 0.001,
+      "loss": 17.959799194335936,
+      "step": 180
+    },
+    {
+      "epoch": 0.10784578053383662,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.001,
+      "loss": 17.317408752441406,
+      "step": 200
+    },
+    {
+      "epoch": 0.11863035858722028,
+      "grad_norm": 3.484375,
+      "learning_rate": 0.001,
+      "loss": 16.665341186523438,
+      "step": 220
+    },
+    {
+      "epoch": 0.12941493664060394,
+      "grad_norm": 4.0625,
+      "learning_rate": 0.001,
+      "loss": 16.151258850097655,
+      "step": 240
+    },
+    {
+      "epoch": 0.1401995146939876,
+      "grad_norm": 2.34375,
+      "learning_rate": 0.001,
+      "loss": 15.6710205078125,
+      "step": 260
+    },
+    {
+      "epoch": 0.15098409274737126,
+      "grad_norm": 2.984375,
+      "learning_rate": 0.001,
+      "loss": 15.264056396484374,
+      "step": 280
+    },
+    {
+      "epoch": 0.1617686708007549,
+      "grad_norm": 1.8125,
+      "learning_rate": 0.001,
+      "loss": 14.908853149414062,
+      "step": 300
+    },
+    {
+      "epoch": 0.17255324885413859,
+      "grad_norm": 2.390625,
+      "learning_rate": 0.001,
+      "loss": 14.611824035644531,
+      "step": 320
+    },
+    {
+      "epoch": 0.18333782690752223,
+      "grad_norm": 1.6484375,
+      "learning_rate": 0.001,
+      "loss": 14.270545959472656,
+      "step": 340
+    },
+    {
+      "epoch": 0.1941224049609059,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.001,
+      "loss": 13.935658264160157,
+      "step": 360
+    },
+    {
+      "epoch": 0.20490698301428956,
+      "grad_norm": 2.75,
+      "learning_rate": 0.001,
+      "loss": 13.626214599609375,
+      "step": 380
+    },
+    {
+      "epoch": 0.21569156106767323,
+      "grad_norm": 2.6875,
+      "learning_rate": 0.001,
+      "loss": 13.423133850097656,
+      "step": 400
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2168126334566400.0,
+  "train_batch_size": 128,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bda593b8a69e45e9f48a5e12fceefb418c681431f4313d1ee1004dc70bcfa0dd
+size 4920

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-500/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu-waleed10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "mlp",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/mlp-silu-waleed10-21L_run/checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed355b33e04badadbc8734b45e04bcc7dcb258518654b97927e25e4219617980
+size 7959296