Training checkpoint at step 1000

trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 1000,
-  "best_metric": 2.493894577026367,
+  "best_metric": 2.491666078567505,
   "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/meta-llama/Llama-3.2-1B/linear_adamw_wd1e-03_7x1024_mem32_bs64_hf_armt_dmem64/run_30/checkpoint-1000",
   "epoch": 0.02,
   "eval_steps": 100,
@@ -11,362 +11,362 @@
   "log_history": [
     {
       "epoch": 0.0005,
-      "grad_norm": 1.374199618670654,
+      "grad_norm": 2.7020849153690363,
       "learning_rate": 4.8e-08,
-      "loss": 3.8192,
+      "loss": 4.4151,
       "step": 25
     },
     {
       "epoch": 0.001,
-      "grad_norm": 1.0626796430084513,
+      "grad_norm": 1.7714713388908587,
       "learning_rate": 9.8e-08,
-      "loss": 3.7481,
+      "loss": 4.2692,
       "step": 50
     },
     {
       "epoch": 0.0015,
-      "grad_norm": 0.554900729210382,
+      "grad_norm": 1.0569441206778722,
       "learning_rate": 1.4800000000000003e-07,
-      "loss": 3.5249,
+      "loss": 3.9071,
       "step": 75
     },
     {
       "epoch": 0.002,
-      "grad_norm": 0.407340177021375,
+      "grad_norm": 0.43396234020605096,
       "learning_rate": 1.9800000000000003e-07,
-      "loss": 3.3424,
+      "loss": 3.4257,
       "step": 100
     },
     {
       "epoch": 0.002,
-      "eval_loss": 3.072638988494873,
-      "eval_runtime": 33.0855,
-      "eval_samples_per_second": 3.536,
-      "eval_steps_per_second": 1.783,
+      "eval_loss": 3.1404547691345215,
+      "eval_runtime": 33.0346,
+      "eval_samples_per_second": 3.542,
+      "eval_steps_per_second": 1.786,
       "step": 100
     },
     {
       "epoch": 0.0025,
-      "grad_norm": 0.22554288925827917,
+      "grad_norm": 0.2585925841488232,
       "learning_rate": 2.48e-07,
-      "loss": 3.1284,
+      "loss": 3.1579,
       "step": 125
     },
     {
       "epoch": 0.003,
-      "grad_norm": 0.14377524138618566,
+      "grad_norm": 0.15143157149805395,
       "learning_rate": 2.9800000000000005e-07,
-      "loss": 2.9699,
+      "loss": 2.9738,
       "step": 150
     },
     {
       "epoch": 0.0035,
-      "grad_norm": 0.1273100096023732,
+      "grad_norm": 0.1080278835540699,
       "learning_rate": 3.48e-07,
-      "loss": 2.8784,
+      "loss": 2.8727,
       "step": 175
     },
     {
       "epoch": 0.004,
-      "grad_norm": 0.08052145135761532,
+      "grad_norm": 0.09698869766195446,
       "learning_rate": 3.9800000000000004e-07,
-      "loss": 2.808,
+      "loss": 2.827,
       "step": 200
     },
     {
       "epoch": 0.004,
-      "eval_loss": 2.7464711666107178,
-      "eval_runtime": 33.2528,
-      "eval_samples_per_second": 3.519,
-      "eval_steps_per_second": 1.774,
+      "eval_loss": 2.752889633178711,
+      "eval_runtime": 33.2978,
+      "eval_samples_per_second": 3.514,
+      "eval_steps_per_second": 1.772,
       "step": 200
     },
     {
       "epoch": 0.0045,
-      "grad_norm": 0.09601426553193865,
+      "grad_norm": 0.07244257780140247,
       "learning_rate": 4.4800000000000004e-07,
-      "loss": 2.7735,
+      "loss": 2.773,
       "step": 225
     },
     {
       "epoch": 0.005,
-      "grad_norm": 0.06772677130883735,
+      "grad_norm": 0.06884256698341579,
       "learning_rate": 4.98e-07,
-      "loss": 2.7358,
+      "loss": 2.7357,
       "step": 250
     },
     {
       "epoch": 0.0055,
-      "grad_norm": 0.0617749171010752,
+      "grad_norm": 0.097508726424133,
       "learning_rate": 5.480000000000001e-07,
-      "loss": 2.7143,
+      "loss": 2.708,
       "step": 275
     },
     {
       "epoch": 0.006,
-      "grad_norm": 0.06081364438555446,
+      "grad_norm": 0.05004869412393589,
       "learning_rate": 5.98e-07,
-      "loss": 2.695,
+      "loss": 2.6786,
       "step": 300
     },
     {
       "epoch": 0.006,
-      "eval_loss": 2.658987045288086,
-      "eval_runtime": 33.098,
-      "eval_samples_per_second": 3.535,
-      "eval_steps_per_second": 1.783,
+      "eval_loss": 2.647900342941284,
+      "eval_runtime": 33.2991,
+      "eval_samples_per_second": 3.514,
+      "eval_steps_per_second": 1.772,
       "step": 300
     },
     {
       "epoch": 0.0065,
-      "grad_norm": 0.055229056926588416,
+      "grad_norm": 0.049132610170334615,
       "learning_rate": 6.48e-07,
-      "loss": 2.6775,
+      "loss": 2.6474,
       "step": 325
     },
     {
       "epoch": 0.007,
-      "grad_norm": 0.05285547880508365,
+      "grad_norm": 0.05729778967483004,
       "learning_rate": 6.98e-07,
-      "loss": 2.6498,
+      "loss": 2.6357,
       "step": 350
     },
     {
       "epoch": 0.0075,
-      "grad_norm": 0.05321418646538081,
+      "grad_norm": 0.04063098299936942,
       "learning_rate": 7.480000000000001e-07,
-      "loss": 2.6375,
+      "loss": 2.6253,
       "step": 375
     },
     {
       "epoch": 0.008,
-      "grad_norm": 0.046163922554101317,
+      "grad_norm": 0.034498073897365616,
       "learning_rate": 7.98e-07,
-      "loss": 2.6273,
+      "loss": 2.6076,
       "step": 400
     },
     {
       "epoch": 0.008,
-      "eval_loss": 2.5985612869262695,
-      "eval_runtime": 33.8223,
-      "eval_samples_per_second": 3.459,
-      "eval_steps_per_second": 1.744,
+      "eval_loss": 2.594409942626953,
+      "eval_runtime": 34.2954,
+      "eval_samples_per_second": 3.412,
+      "eval_steps_per_second": 1.72,
       "step": 400
     },
     {
       "epoch": 0.0085,
-      "grad_norm": 0.05184119325112733,
+      "grad_norm": 0.03558666298221422,
       "learning_rate": 8.480000000000001e-07,
-      "loss": 2.6037,
+      "loss": 2.595,
       "step": 425
     },
     {
       "epoch": 0.009,
-      "grad_norm": 0.036013105129600216,
+      "grad_norm": 0.03443154792307346,
       "learning_rate": 8.980000000000001e-07,
-      "loss": 2.594,
+      "loss": 2.5908,
       "step": 450
     },
     {
       "epoch": 0.0095,
-      "grad_norm": 0.03061363860030697,
+      "grad_norm": 0.035488270944549226,
       "learning_rate": 9.480000000000001e-07,
-      "loss": 2.5746,
+      "loss": 2.5809,
       "step": 475
     },
     {
       "epoch": 0.01,
-      "grad_norm": 0.036970324693471876,
+      "grad_norm": 0.03146469333633836,
       "learning_rate": 9.98e-07,
-      "loss": 2.5827,
+      "loss": 2.5736,
       "step": 500
     },
     {
       "epoch": 0.01,
-      "eval_loss": 2.56477952003479,
-      "eval_runtime": 33.0671,
-      "eval_samples_per_second": 3.538,
-      "eval_steps_per_second": 1.784,
+      "eval_loss": 2.5649118423461914,
+      "eval_runtime": 36.3209,
+      "eval_samples_per_second": 3.221,
+      "eval_steps_per_second": 1.624,
       "step": 500
     },
     {
       "epoch": 0.0105,
-      "grad_norm": 0.03044033343054017,
+      "grad_norm": 0.03114420601995518,
       "learning_rate": 1.0480000000000002e-06,
-      "loss": 2.5717,
+      "loss": 2.5671,
       "step": 525
     },
     {
       "epoch": 0.011,
-      "grad_norm": 0.03340669717167394,
+      "grad_norm": 0.03096542621853569,
       "learning_rate": 1.0980000000000001e-06,
-      "loss": 2.5613,
+      "loss": 2.5558,
       "step": 550
     },
     {
       "epoch": 0.0115,
-      "grad_norm": 0.029909971322257495,
+      "grad_norm": 0.02905782871061764,
       "learning_rate": 1.148e-06,
-      "loss": 2.5661,
+      "loss": 2.5623,
       "step": 575
     },
     {
       "epoch": 0.012,
-      "grad_norm": 0.028901093383770705,
+      "grad_norm": 0.030562740052257713,
       "learning_rate": 1.1980000000000002e-06,
-      "loss": 2.561,
+      "loss": 2.5622,
       "step": 600
     },
     {
       "epoch": 0.012,
-      "eval_loss": 2.5446865558624268,
-      "eval_runtime": 33.1445,
-      "eval_samples_per_second": 3.53,
-      "eval_steps_per_second": 1.78,
+      "eval_loss": 2.54438853263855,
+      "eval_runtime": 36.9048,
+      "eval_samples_per_second": 3.17,
+      "eval_steps_per_second": 1.599,
       "step": 600
     },
     {
       "epoch": 0.0125,
-      "grad_norm": 0.0338582199400455,
+      "grad_norm": 0.03153013886632261,
       "learning_rate": 1.248e-06,
-      "loss": 2.5392,
+      "loss": 2.5404,
       "step": 625
     },
     {
       "epoch": 0.013,
-      "grad_norm": 0.031107046362937877,
+      "grad_norm": 0.028752715448972253,
       "learning_rate": 1.2980000000000001e-06,
-      "loss": 2.5378,
+      "loss": 2.5396,
       "step": 650
     },
     {
       "epoch": 0.0135,
-      "grad_norm": 0.027777474622611625,
+      "grad_norm": 0.03011440752674912,
       "learning_rate": 1.348e-06,
-      "loss": 2.5421,
+      "loss": 2.5418,
       "step": 675
     },
     {
       "epoch": 0.014,
-      "grad_norm": 0.032848272502404616,
+      "grad_norm": 0.027934694405631223,
       "learning_rate": 1.3980000000000002e-06,
-      "loss": 2.5345,
+      "loss": 2.5371,
       "step": 700
     },
     {
       "epoch": 0.014,
-      "eval_loss": 2.5294137001037598,
-      "eval_runtime": 33.222,
-      "eval_samples_per_second": 3.522,
-      "eval_steps_per_second": 1.776,
+      "eval_loss": 2.529193878173828,
+      "eval_runtime": 41.1238,
+      "eval_samples_per_second": 2.845,
+      "eval_steps_per_second": 1.435,
       "step": 700
     },
     {
       "epoch": 0.0145,
-      "grad_norm": 0.028260965292318807,
+      "grad_norm": 0.029595976023437127,
       "learning_rate": 1.4480000000000002e-06,
-      "loss": 2.5342,
+      "loss": 2.5285,
       "step": 725
     },
     {
       "epoch": 0.015,
-      "grad_norm": 0.02748431921263886,
+      "grad_norm": 0.03539675413826323,
       "learning_rate": 1.498e-06,
-      "loss": 2.5188,
+      "loss": 2.5286,
       "step": 750
     },
     {
       "epoch": 0.0155,
-      "grad_norm": 0.02859453618814513,
+      "grad_norm": 0.03363430055312599,
       "learning_rate": 1.548e-06,
-      "loss": 2.5245,
+      "loss": 2.5146,
       "step": 775
     },
     {
       "epoch": 0.016,
-      "grad_norm": 0.03906649589898274,
+      "grad_norm": 0.036143105087014814,
       "learning_rate": 1.5980000000000002e-06,
-      "loss": 2.5142,
+      "loss": 2.505,
       "step": 800
     },
     {
       "epoch": 0.016,
-      "eval_loss": 2.5156726837158203,
-      "eval_runtime": 33.2465,
-      "eval_samples_per_second": 3.519,
-      "eval_steps_per_second": 1.775,
+      "eval_loss": 2.5139412879943848,
+      "eval_runtime": 41.1683,
+      "eval_samples_per_second": 2.842,
+      "eval_steps_per_second": 1.433,
       "step": 800
     },
     {
       "epoch": 0.0165,
-      "grad_norm": 0.03055728445213633,
+      "grad_norm": 0.07023101199739942,
       "learning_rate": 1.6480000000000001e-06,
-      "loss": 2.504,
+      "loss": 2.5168,
       "step": 825
     },
     {
       "epoch": 0.017,
-      "grad_norm": 0.03831919004049627,
+      "grad_norm": 0.036491334947115234,
       "learning_rate": 1.6980000000000003e-06,
-      "loss": 2.5096,
+      "loss": 2.5088,
       "step": 850
     },
     {
       "epoch": 0.0175,
-      "grad_norm": 0.04714764947462498,
+      "grad_norm": 0.052452197634130976,
       "learning_rate": 1.7480000000000002e-06,
-      "loss": 2.5057,
+      "loss": 2.4912,
       "step": 875
     },
     {
       "epoch": 0.018,
-      "grad_norm": 0.04480333925801958,
+      "grad_norm": 0.04670910122190947,
       "learning_rate": 1.798e-06,
-      "loss": 2.4949,
+      "loss": 2.4976,
       "step": 900
     },
     {
       "epoch": 0.018,
-      "eval_loss": 2.503145456314087,
-      "eval_runtime": 33.3398,
-      "eval_samples_per_second": 3.509,
-      "eval_steps_per_second": 1.77,
+      "eval_loss": 2.5007853507995605,
+      "eval_runtime": 38.6574,
+      "eval_samples_per_second": 3.027,
+      "eval_steps_per_second": 1.526,
       "step": 900
     },
     {
       "epoch": 0.0185,
-      "grad_norm": 0.04514734300904146,
+      "grad_norm": 0.054668821609945045,
       "learning_rate": 1.8480000000000001e-06,
-      "loss": 2.5044,
+      "loss": 2.4915,
       "step": 925
     },
     {
       "epoch": 0.019,
-      "grad_norm": 0.03664477032679196,
+      "grad_norm": 0.048767134507416036,
       "learning_rate": 1.898e-06,
-      "loss": 2.4857,
+      "loss": 2.488,
       "step": 950
     },
     {
       "epoch": 0.0195,
-      "grad_norm": 0.03891788038244039,
+      "grad_norm": 0.04356515009200933,
       "learning_rate": 1.9480000000000002e-06,
-      "loss": 2.4954,
+      "loss": 2.4904,
       "step": 975
     },
     {
       "epoch": 0.02,
-      "grad_norm": 0.041701680819843504,
+      "grad_norm": 0.0393758269000268,
       "learning_rate": 1.998e-06,
-      "loss": 2.4935,
+      "loss": 2.4859,
       "step": 1000
     },
     {
       "epoch": 0.02,
-      "eval_loss": 2.493894577026367,
-      "eval_runtime": 33.4036,
-      "eval_samples_per_second": 3.503,
-      "eval_steps_per_second": 1.766,
+      "eval_loss": 2.491666078567505,
+      "eval_runtime": 40.9073,
+      "eval_samples_per_second": 2.86,
+      "eval_steps_per_second": 1.442,
       "step": 1000
     }
   ],