Training checkpoint at step 6500

trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 6000,
-  "best_metric": 2.594125509262085,
-  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-6000",
-  "epoch": 0.12,
+  "best_global_step": 6500,
+  "best_metric": 2.585561990737915,
+  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-6500",
+  "epoch": 0.13,
   "eval_steps": 100,
-  "global_step": 6000,
+  "global_step": 6500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2168,6 +2168,186 @@
       "eval_samples_per_second": 2.417,
       "eval_steps_per_second": 1.208,
       "step": 6000
+    },
+    {
+      "epoch": 0.1205,
+      "grad_norm": 2.1932467610128166,
+      "learning_rate": 9.772444444444445e-06,
+      "loss": 2.5892,
+      "step": 6025
+    },
+    {
+      "epoch": 0.121,
+      "grad_norm": 2.387425729108963,
+      "learning_rate": 9.76688888888889e-06,
+      "loss": 2.5918,
+      "step": 6050
+    },
+    {
+      "epoch": 0.1215,
+      "grad_norm": 2.8624591702116313,
+      "learning_rate": 9.761333333333334e-06,
+      "loss": 2.5875,
+      "step": 6075
+    },
+    {
+      "epoch": 0.122,
+      "grad_norm": 2.930012610934339,
+      "learning_rate": 9.755777777777778e-06,
+      "loss": 2.5906,
+      "step": 6100
+    },
+    {
+      "epoch": 0.122,
+      "eval_loss": 2.592097282409668,
+      "eval_runtime": 42.1118,
+      "eval_samples_per_second": 2.47,
+      "eval_steps_per_second": 1.235,
+      "step": 6100
+    },
+    {
+      "epoch": 0.1225,
+      "grad_norm": 3.6585883804987596,
+      "learning_rate": 9.750222222222223e-06,
+      "loss": 2.5888,
+      "step": 6125
+    },
+    {
+      "epoch": 0.123,
+      "grad_norm": 2.9636602337569213,
+      "learning_rate": 9.744666666666668e-06,
+      "loss": 2.5848,
+      "step": 6150
+    },
+    {
+      "epoch": 0.1235,
+      "grad_norm": 2.6452546886265242,
+      "learning_rate": 9.739111111111112e-06,
+      "loss": 2.5875,
+      "step": 6175
+    },
+    {
+      "epoch": 0.124,
+      "grad_norm": 2.230890007256631,
+      "learning_rate": 9.733555555555555e-06,
+      "loss": 2.5928,
+      "step": 6200
+    },
+    {
+      "epoch": 0.124,
+      "eval_loss": 2.591871976852417,
+      "eval_runtime": 42.2393,
+      "eval_samples_per_second": 2.462,
+      "eval_steps_per_second": 1.231,
+      "step": 6200
+    },
+    {
+      "epoch": 0.1245,
+      "grad_norm": 2.2263966783946643,
+      "learning_rate": 9.728e-06,
+      "loss": 2.5913,
+      "step": 6225
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 3.0917521864623168,
+      "learning_rate": 9.722444444444446e-06,
+      "loss": 2.5858,
+      "step": 6250
+    },
+    {
+      "epoch": 0.1255,
+      "grad_norm": 3.406162518240377,
+      "learning_rate": 9.71688888888889e-06,
+      "loss": 2.5824,
+      "step": 6275
+    },
+    {
+      "epoch": 0.126,
+      "grad_norm": 1.9288658675383707,
+      "learning_rate": 9.711333333333333e-06,
+      "loss": 2.5881,
+      "step": 6300
+    },
+    {
+      "epoch": 0.126,
+      "eval_loss": 2.588792085647583,
+      "eval_runtime": 42.1993,
+      "eval_samples_per_second": 2.464,
+      "eval_steps_per_second": 1.232,
+      "step": 6300
+    },
+    {
+      "epoch": 0.1265,
+      "grad_norm": 2.3054152552517557,
+      "learning_rate": 9.705777777777778e-06,
+      "loss": 2.5777,
+      "step": 6325
+    },
+    {
+      "epoch": 0.127,
+      "grad_norm": 2.4215099152732438,
+      "learning_rate": 9.700222222222224e-06,
+      "loss": 2.5905,
+      "step": 6350
+    },
+    {
+      "epoch": 0.1275,
+      "grad_norm": 2.1008082850001584,
+      "learning_rate": 9.694666666666667e-06,
+      "loss": 2.5891,
+      "step": 6375
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 2.548161937775528,
+      "learning_rate": 9.68911111111111e-06,
+      "loss": 2.5828,
+      "step": 6400
+    },
+    {
+      "epoch": 0.128,
+      "eval_loss": 2.588566780090332,
+      "eval_runtime": 42.2757,
+      "eval_samples_per_second": 2.46,
+      "eval_steps_per_second": 1.23,
+      "step": 6400
+    },
+    {
+      "epoch": 0.1285,
+      "grad_norm": 2.1721864313913555,
+      "learning_rate": 9.683555555555556e-06,
+      "loss": 2.585,
+      "step": 6425
+    },
+    {
+      "epoch": 0.129,
+      "grad_norm": 2.6656100643358567,
+      "learning_rate": 9.678000000000001e-06,
+      "loss": 2.5859,
+      "step": 6450
+    },
+    {
+      "epoch": 0.1295,
+      "grad_norm": 2.14442087538069,
+      "learning_rate": 9.672444444444445e-06,
+      "loss": 2.5897,
+      "step": 6475
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 2.544695719649347,
+      "learning_rate": 9.66688888888889e-06,
+      "loss": 2.5819,
+      "step": 6500
+    },
+    {
+      "epoch": 0.13,
+      "eval_loss": 2.585561990737915,
+      "eval_runtime": 42.2362,
+      "eval_samples_per_second": 2.462,
+      "eval_steps_per_second": 1.231,
+      "step": 6500
     }
   ],
   "logging_steps": 25,
@@ -2187,7 +2367,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.3465316020260438e+19,
+  "total_flos": 1.4587425704363688e+19,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null