Training checkpoint at step 1000

trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 1000,
-  "best_metric": 2.491666078567505,
+  "best_metric": 2.4966063499450684,
   "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/meta-llama/Llama-3.2-1B/linear_adamw_wd1e-03_7x1024_mem32_bs64_hf_armt_dmem64/run_30/checkpoint-1000",
   "epoch": 0.02,
   "eval_steps": 100,
@@ -11,362 +11,362 @@
   "log_history": [
     {
       "epoch": 0.0005,
-      "grad_norm": 2.7020849153690363,
+      "grad_norm": 2.624103276270124,
       "learning_rate": 4.8e-08,
-      "loss": 4.4151,
+      "loss": 4.0893,
       "step": 25
     },
     {
       "epoch": 0.001,
-      "grad_norm": 1.7714713388908587,
+      "grad_norm": 1.3629568986234561,
       "learning_rate": 9.8e-08,
-      "loss": 4.2692,
+      "loss": 3.9543,
       "step": 50
     },
     {
       "epoch": 0.0015,
-      "grad_norm": 1.0569441206778722,
+      "grad_norm": 0.8050128701430977,
       "learning_rate": 1.4800000000000003e-07,
-      "loss": 3.9071,
+      "loss": 3.6763,
       "step": 75
     },
     {
       "epoch": 0.002,
-      "grad_norm": 0.43396234020605096,
+      "grad_norm": 0.3690286383727022,
       "learning_rate": 1.9800000000000003e-07,
-      "loss": 3.4257,
+      "loss": 3.327,
       "step": 100
     },
     {
       "epoch": 0.002,
-      "eval_loss": 3.1404547691345215,
-      "eval_runtime": 33.0346,
-      "eval_samples_per_second": 3.542,
-      "eval_steps_per_second": 1.786,
+      "eval_loss": 3.100055694580078,
+      "eval_runtime": 32.7706,
+      "eval_samples_per_second": 3.57,
+      "eval_steps_per_second": 1.8,
       "step": 100
     },
     {
       "epoch": 0.0025,
-      "grad_norm": 0.2585925841488232,
+      "grad_norm": 0.24011694167100578,
       "learning_rate": 2.48e-07,
-      "loss": 3.1579,
+      "loss": 3.1322,
       "step": 125
     },
     {
       "epoch": 0.003,
-      "grad_norm": 0.15143157149805395,
+      "grad_norm": 0.149511940963387,
       "learning_rate": 2.9800000000000005e-07,
-      "loss": 2.9738,
+      "loss": 2.9672,
       "step": 150
     },
     {
       "epoch": 0.0035,
-      "grad_norm": 0.1080278835540699,
+      "grad_norm": 0.10071711520195754,
       "learning_rate": 3.48e-07,
-      "loss": 2.8727,
+      "loss": 2.8684,
       "step": 175
     },
     {
       "epoch": 0.004,
-      "grad_norm": 0.09698869766195446,
+      "grad_norm": 0.09695377414070089,
       "learning_rate": 3.9800000000000004e-07,
-      "loss": 2.827,
+      "loss": 2.8244,
       "step": 200
     },
     {
       "epoch": 0.004,
-      "eval_loss": 2.752889633178711,
-      "eval_runtime": 33.2978,
-      "eval_samples_per_second": 3.514,
-      "eval_steps_per_second": 1.772,
+      "eval_loss": 2.7518060207366943,
+      "eval_runtime": 32.9203,
+      "eval_samples_per_second": 3.554,
+      "eval_steps_per_second": 1.792,
       "step": 200
     },
     {
       "epoch": 0.0045,
-      "grad_norm": 0.07244257780140247,
+      "grad_norm": 0.06541174981920718,
       "learning_rate": 4.4800000000000004e-07,
-      "loss": 2.773,
+      "loss": 2.7736,
       "step": 225
     },
     {
       "epoch": 0.005,
-      "grad_norm": 0.06884256698341579,
+      "grad_norm": 0.061297886999798934,
       "learning_rate": 4.98e-07,
-      "loss": 2.7357,
+      "loss": 2.7392,
       "step": 250
     },
     {
       "epoch": 0.0055,
-      "grad_norm": 0.097508726424133,
+      "grad_norm": 0.07881073149840945,
       "learning_rate": 5.480000000000001e-07,
-      "loss": 2.708,
+      "loss": 2.7194,
       "step": 275
     },
     {
       "epoch": 0.006,
-      "grad_norm": 0.05004869412393589,
+      "grad_norm": 0.05125386617161651,
       "learning_rate": 5.98e-07,
-      "loss": 2.6786,
+      "loss": 2.6982,
       "step": 300
     },
     {
       "epoch": 0.006,
-      "eval_loss": 2.647900342941284,
-      "eval_runtime": 33.2991,
-      "eval_samples_per_second": 3.514,
-      "eval_steps_per_second": 1.772,
+      "eval_loss": 2.6622018814086914,
+      "eval_runtime": 32.9076,
+      "eval_samples_per_second": 3.555,
+      "eval_steps_per_second": 1.793,
       "step": 300
     },
     {
       "epoch": 0.0065,
-      "grad_norm": 0.049132610170334615,
+      "grad_norm": 0.04659366450077996,
       "learning_rate": 6.48e-07,
-      "loss": 2.6474,
+      "loss": 2.6725,
       "step": 325
     },
     {
       "epoch": 0.007,
-      "grad_norm": 0.05729778967483004,
+      "grad_norm": 0.04588097652548341,
       "learning_rate": 6.98e-07,
-      "loss": 2.6357,
+      "loss": 2.6592,
       "step": 350
     },
     {
       "epoch": 0.0075,
-      "grad_norm": 0.04063098299936942,
+      "grad_norm": 0.058421958212028904,
       "learning_rate": 7.480000000000001e-07,
-      "loss": 2.6253,
+      "loss": 2.6481,
       "step": 375
     },
     {
       "epoch": 0.008,
-      "grad_norm": 0.034498073897365616,
+      "grad_norm": 0.04289575736155661,
       "learning_rate": 7.98e-07,
-      "loss": 2.6076,
+      "loss": 2.6257,
       "step": 400
     },
     {
       "epoch": 0.008,
-      "eval_loss": 2.594409942626953,
-      "eval_runtime": 34.2954,
-      "eval_samples_per_second": 3.412,
-      "eval_steps_per_second": 1.72,
+      "eval_loss": 2.6052613258361816,
+      "eval_runtime": 32.8227,
+      "eval_samples_per_second": 3.565,
+      "eval_steps_per_second": 1.798,
       "step": 400
     },
     {
       "epoch": 0.0085,
-      "grad_norm": 0.03558666298221422,
+      "grad_norm": 0.041602666338794385,
       "learning_rate": 8.480000000000001e-07,
-      "loss": 2.595,
+      "loss": 2.6089,
       "step": 425
     },
     {
       "epoch": 0.009,
-      "grad_norm": 0.03443154792307346,
+      "grad_norm": 0.040090024026539266,
       "learning_rate": 8.980000000000001e-07,
-      "loss": 2.5908,
+      "loss": 2.5985,
       "step": 450
     },
     {
       "epoch": 0.0095,
-      "grad_norm": 0.035488270944549226,
+      "grad_norm": 0.05346463020318845,
       "learning_rate": 9.480000000000001e-07,
-      "loss": 2.5809,
+      "loss": 2.5858,
       "step": 475
     },
     {
       "epoch": 0.01,
-      "grad_norm": 0.03146469333633836,
+      "grad_norm": 0.03240197247016216,
       "learning_rate": 9.98e-07,
-      "loss": 2.5736,
+      "loss": 2.5773,
       "step": 500
     },
     {
       "epoch": 0.01,
-      "eval_loss": 2.5649118423461914,
-      "eval_runtime": 36.3209,
-      "eval_samples_per_second": 3.221,
-      "eval_steps_per_second": 1.624,
+      "eval_loss": 2.5677218437194824,
+      "eval_runtime": 32.9146,
+      "eval_samples_per_second": 3.555,
+      "eval_steps_per_second": 1.793,
       "step": 500
     },
     {
       "epoch": 0.0105,
-      "grad_norm": 0.03114420601995518,
+      "grad_norm": 0.030627609315729644,
       "learning_rate": 1.0480000000000002e-06,
-      "loss": 2.5671,
+      "loss": 2.5695,
       "step": 525
     },
     {
       "epoch": 0.011,
-      "grad_norm": 0.03096542621853569,
+      "grad_norm": 0.03146801435404312,
       "learning_rate": 1.0980000000000001e-06,
-      "loss": 2.5558,
+      "loss": 2.558,
       "step": 550
     },
     {
       "epoch": 0.0115,
-      "grad_norm": 0.02905782871061764,
+      "grad_norm": 0.028453864143727626,
       "learning_rate": 1.148e-06,
-      "loss": 2.5623,
+      "loss": 2.5645,
       "step": 575
     },
     {
       "epoch": 0.012,
-      "grad_norm": 0.030562740052257713,
+      "grad_norm": 0.03026805511159676,
       "learning_rate": 1.1980000000000002e-06,
-      "loss": 2.5622,
+      "loss": 2.5645,
       "step": 600
     },
     {
       "epoch": 0.012,
-      "eval_loss": 2.54438853263855,
-      "eval_runtime": 36.9048,
-      "eval_samples_per_second": 3.17,
-      "eval_steps_per_second": 1.599,
+      "eval_loss": 2.546586275100708,
+      "eval_runtime": 32.8424,
+      "eval_samples_per_second": 3.562,
+      "eval_steps_per_second": 1.796,
       "step": 600
     },
     {
       "epoch": 0.0125,
-      "grad_norm": 0.03153013886632261,
+      "grad_norm": 0.032033771539522,
       "learning_rate": 1.248e-06,
-      "loss": 2.5404,
+      "loss": 2.5424,
       "step": 625
     },
     {
       "epoch": 0.013,
-      "grad_norm": 0.028752715448972253,
+      "grad_norm": 0.0281966122475446,
       "learning_rate": 1.2980000000000001e-06,
-      "loss": 2.5396,
+      "loss": 2.5409,
       "step": 650
     },
     {
       "epoch": 0.0135,
-      "grad_norm": 0.03011440752674912,
+      "grad_norm": 0.02887428243284281,
       "learning_rate": 1.348e-06,
-      "loss": 2.5418,
+      "loss": 2.543,
       "step": 675
     },
     {
       "epoch": 0.014,
-      "grad_norm": 0.027934694405631223,
+      "grad_norm": 0.027672621753278132,
       "learning_rate": 1.3980000000000002e-06,
-      "loss": 2.5371,
+      "loss": 2.5385,
       "step": 700
     },
     {
       "epoch": 0.014,
-      "eval_loss": 2.529193878173828,
-      "eval_runtime": 41.1238,
-      "eval_samples_per_second": 2.845,
-      "eval_steps_per_second": 1.435,
+      "eval_loss": 2.530237913131714,
+      "eval_runtime": 32.7994,
+      "eval_samples_per_second": 3.567,
+      "eval_steps_per_second": 1.799,
       "step": 700
     },
     {
       "epoch": 0.0145,
-      "grad_norm": 0.029595976023437127,
+      "grad_norm": 0.030815191380069624,
       "learning_rate": 1.4480000000000002e-06,
-      "loss": 2.5285,
+      "loss": 2.5302,
       "step": 725
     },
     {
       "epoch": 0.015,
-      "grad_norm": 0.03539675413826323,
+      "grad_norm": 0.0336387385604783,
       "learning_rate": 1.498e-06,
-      "loss": 2.5286,
+      "loss": 2.531,
       "step": 750
     },
     {
       "epoch": 0.0155,
-      "grad_norm": 0.03363430055312599,
+      "grad_norm": 0.02858543320323233,
       "learning_rate": 1.548e-06,
-      "loss": 2.5146,
+      "loss": 2.5184,
       "step": 775
     },
     {
       "epoch": 0.016,
-      "grad_norm": 0.036143105087014814,
+      "grad_norm": 0.028120393653995705,
       "learning_rate": 1.5980000000000002e-06,
-      "loss": 2.505,
+      "loss": 2.5101,
       "step": 800
     },
     {
       "epoch": 0.016,
-      "eval_loss": 2.5139412879943848,
-      "eval_runtime": 41.1683,
-      "eval_samples_per_second": 2.842,
-      "eval_steps_per_second": 1.433,
+      "eval_loss": 2.5182888507843018,
+      "eval_runtime": 33.2135,
+      "eval_samples_per_second": 3.523,
+      "eval_steps_per_second": 1.776,
       "step": 800
     },
     {
       "epoch": 0.0165,
-      "grad_norm": 0.07023101199739942,
+      "grad_norm": 0.03014167593156162,
       "learning_rate": 1.6480000000000001e-06,
-      "loss": 2.5168,
+      "loss": 2.5232,
       "step": 825
     },
     {
       "epoch": 0.017,
-      "grad_norm": 0.036491334947115234,
+      "grad_norm": 0.028528349033195077,
       "learning_rate": 1.6980000000000003e-06,
-      "loss": 2.5088,
+      "loss": 2.5162,
       "step": 850
     },
     {
       "epoch": 0.0175,
-      "grad_norm": 0.052452197634130976,
+      "grad_norm": 0.031230193601244804,
       "learning_rate": 1.7480000000000002e-06,
-      "loss": 2.4912,
+      "loss": 2.4995,
       "step": 875
     },
     {
       "epoch": 0.018,
-      "grad_norm": 0.04670910122190947,
+      "grad_norm": 0.03555060954716827,
       "learning_rate": 1.798e-06,
-      "loss": 2.4976,
+      "loss": 2.5064,
       "step": 900
     },
     {
       "epoch": 0.018,
-      "eval_loss": 2.5007853507995605,
-      "eval_runtime": 38.6574,
-      "eval_samples_per_second": 3.027,
-      "eval_steps_per_second": 1.526,
+      "eval_loss": 2.5070879459381104,
+      "eval_runtime": 33.3807,
+      "eval_samples_per_second": 3.505,
+      "eval_steps_per_second": 1.767,
       "step": 900
     },
     {
       "epoch": 0.0185,
-      "grad_norm": 0.054668821609945045,
+      "grad_norm": 0.03561871969060444,
       "learning_rate": 1.8480000000000001e-06,
-      "loss": 2.4915,
+      "loss": 2.5004,
       "step": 925
     },
     {
       "epoch": 0.019,
-      "grad_norm": 0.048767134507416036,
+      "grad_norm": 0.03094584673111385,
       "learning_rate": 1.898e-06,
-      "loss": 2.488,
+      "loss": 2.4959,
       "step": 950
     },
     {
       "epoch": 0.0195,
-      "grad_norm": 0.04356515009200933,
+      "grad_norm": 0.035545021685136444,
       "learning_rate": 1.9480000000000002e-06,
-      "loss": 2.4904,
+      "loss": 2.4982,
       "step": 975
     },
     {
       "epoch": 0.02,
-      "grad_norm": 0.0393758269000268,
+      "grad_norm": 0.0370422613473599,
       "learning_rate": 1.998e-06,
-      "loss": 2.4859,
+      "loss": 2.4927,
       "step": 1000
     },
     {
       "epoch": 0.02,
-      "eval_loss": 2.491666078567505,
-      "eval_runtime": 40.9073,
-      "eval_samples_per_second": 2.86,
-      "eval_steps_per_second": 1.442,
+      "eval_loss": 2.4966063499450684,
+      "eval_runtime": 33.3038,
+      "eval_samples_per_second": 3.513,
+      "eval_steps_per_second": 1.772,
       "step": 1000
     }
   ],