Training checkpoint at step 8500

trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 8000,
-  "best_metric": 2.568809986114502,
-  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-8000",
-  "epoch": 0.16,
+  "best_global_step": 8500,
+  "best_metric": 2.564678430557251,
+  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-8500",
+  "epoch": 0.17,
   "eval_steps": 100,
-  "global_step": 8000,
+  "global_step": 8500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2888,6 +2888,186 @@
       "eval_samples_per_second": 2.309,
       "eval_steps_per_second": 1.155,
       "step": 8000
+    },
+    {
+      "epoch": 0.1605,
+      "grad_norm": 2.6560367873629835,
+      "learning_rate": 9.328000000000001e-06,
+      "loss": 2.5588,
+      "step": 8025
+    },
+    {
+      "epoch": 0.161,
+      "grad_norm": 2.2401297319157614,
+      "learning_rate": 9.322444444444445e-06,
+      "loss": 2.564,
+      "step": 8050
+    },
+    {
+      "epoch": 0.1615,
+      "grad_norm": 2.2847898029930653,
+      "learning_rate": 9.31688888888889e-06,
+      "loss": 2.5643,
+      "step": 8075
+    },
+    {
+      "epoch": 0.162,
+      "grad_norm": 2.798251121826375,
+      "learning_rate": 9.311333333333335e-06,
+      "loss": 2.5577,
+      "step": 8100
+    },
+    {
+      "epoch": 0.162,
+      "eval_loss": 2.568058967590332,
+      "eval_runtime": 42.5915,
+      "eval_samples_per_second": 2.442,
+      "eval_steps_per_second": 1.221,
+      "step": 8100
+    },
+    {
+      "epoch": 0.1625,
+      "grad_norm": 2.0139748360698895,
+      "learning_rate": 9.305777777777779e-06,
+      "loss": 2.5716,
+      "step": 8125
+    },
+    {
+      "epoch": 0.163,
+      "grad_norm": 2.052859658987244,
+      "learning_rate": 9.300222222222222e-06,
+      "loss": 2.5555,
+      "step": 8150
+    },
+    {
+      "epoch": 0.1635,
+      "grad_norm": 2.6452792973388584,
+      "learning_rate": 9.294666666666668e-06,
+      "loss": 2.5545,
+      "step": 8175
+    },
+    {
+      "epoch": 0.164,
+      "grad_norm": 2.8085427073848543,
+      "learning_rate": 9.289111111111113e-06,
+      "loss": 2.5575,
+      "step": 8200
+    },
+    {
+      "epoch": 0.164,
+      "eval_loss": 2.56640625,
+      "eval_runtime": 42.2476,
+      "eval_samples_per_second": 2.462,
+      "eval_steps_per_second": 1.231,
+      "step": 8200
+    },
+    {
+      "epoch": 0.1645,
+      "grad_norm": 1.994417686652318,
+      "learning_rate": 9.283555555555556e-06,
+      "loss": 2.5634,
+      "step": 8225
+    },
+    {
+      "epoch": 0.165,
+      "grad_norm": 2.8569259303287917,
+      "learning_rate": 9.278e-06,
+      "loss": 2.5711,
+      "step": 8250
+    },
+    {
+      "epoch": 0.1655,
+      "grad_norm": 2.15031573602464,
+      "learning_rate": 9.272444444444445e-06,
+      "loss": 2.5515,
+      "step": 8275
+    },
+    {
+      "epoch": 0.166,
+      "grad_norm": 2.1903087160864234,
+      "learning_rate": 9.26688888888889e-06,
+      "loss": 2.5588,
+      "step": 8300
+    },
+    {
+      "epoch": 0.166,
+      "eval_loss": 2.565354585647583,
+      "eval_runtime": 42.2533,
+      "eval_samples_per_second": 2.461,
+      "eval_steps_per_second": 1.231,
+      "step": 8300
+    },
+    {
+      "epoch": 0.1665,
+      "grad_norm": 2.1661066402797697,
+      "learning_rate": 9.261333333333334e-06,
+      "loss": 2.5582,
+      "step": 8325
+    },
+    {
+      "epoch": 0.167,
+      "grad_norm": 2.3738673472152603,
+      "learning_rate": 9.25577777777778e-06,
+      "loss": 2.5598,
+      "step": 8350
+    },
+    {
+      "epoch": 0.1675,
+      "grad_norm": 1.893415788443222,
+      "learning_rate": 9.250222222222223e-06,
+      "loss": 2.5553,
+      "step": 8375
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 3.245074933027149,
+      "learning_rate": 9.244666666666668e-06,
+      "loss": 2.5632,
+      "step": 8400
+    },
+    {
+      "epoch": 0.168,
+      "eval_loss": 2.565354585647583,
+      "eval_runtime": 42.2015,
+      "eval_samples_per_second": 2.464,
+      "eval_steps_per_second": 1.232,
+      "step": 8400
+    },
+    {
+      "epoch": 0.1685,
+      "grad_norm": 2.359910509969222,
+      "learning_rate": 9.239111111111112e-06,
+      "loss": 2.5564,
+      "step": 8425
+    },
+    {
+      "epoch": 0.169,
+      "grad_norm": 2.1851033577602355,
+      "learning_rate": 9.233555555555557e-06,
+      "loss": 2.5532,
+      "step": 8450
+    },
+    {
+      "epoch": 0.1695,
+      "grad_norm": 2.0954334474208443,
+      "learning_rate": 9.228e-06,
+      "loss": 2.5585,
+      "step": 8475
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 2.326393982849659,
+      "learning_rate": 9.222444444444446e-06,
+      "loss": 2.5639,
+      "step": 8500
+    },
+    {
+      "epoch": 0.17,
+      "eval_loss": 2.564678430557251,
+      "eval_runtime": 42.3289,
+      "eval_samples_per_second": 2.457,
+      "eval_steps_per_second": 1.228,
+      "step": 8500
     }
   ],
   "logging_steps": 25,
@@ -2907,7 +3087,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.7953754756673438e+19,
+  "total_flos": 1.9075864440776688e+19,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null