Training checkpoint at step 9500

trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 8900,
-  "best_metric": 2.559945821762085,
-  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-8500",
-  "epoch": 0.18,
+  "best_global_step": 9500,
+  "best_metric": 2.555588960647583,
+  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-9500",
+  "epoch": 0.19,
   "eval_steps": 100,
-  "global_step": 9000,
+  "global_step": 9500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3248,6 +3248,186 @@
       "eval_samples_per_second": 2.461,
       "eval_steps_per_second": 1.23,
       "step": 9000
+    },
+    {
+      "epoch": 0.1805,
+      "grad_norm": 2.7335192428299697,
+      "learning_rate": 9.105777777777779e-06,
+      "loss": 2.56,
+      "step": 9025
+    },
+    {
+      "epoch": 0.181,
+      "grad_norm": 2.616833970329197,
+      "learning_rate": 9.100222222222223e-06,
+      "loss": 2.5659,
+      "step": 9050
+    },
+    {
+      "epoch": 0.1815,
+      "grad_norm": 2.636296249975529,
+      "learning_rate": 9.094666666666668e-06,
+      "loss": 2.5605,
+      "step": 9075
+    },
+    {
+      "epoch": 0.182,
+      "grad_norm": 2.1413102875849828,
+      "learning_rate": 9.089111111111111e-06,
+      "loss": 2.5454,
+      "step": 9100
+    },
+    {
+      "epoch": 0.182,
+      "eval_loss": 2.558293342590332,
+      "eval_runtime": 42.2294,
+      "eval_samples_per_second": 2.463,
+      "eval_steps_per_second": 1.231,
+      "step": 9100
+    },
+    {
+      "epoch": 0.1825,
+      "grad_norm": 2.195374313863304,
+      "learning_rate": 9.083555555555557e-06,
+      "loss": 2.5584,
+      "step": 9125
+    },
+    {
+      "epoch": 0.183,
+      "grad_norm": 2.9470418486379546,
+      "learning_rate": 9.078000000000002e-06,
+      "loss": 2.5604,
+      "step": 9150
+    },
+    {
+      "epoch": 0.1835,
+      "grad_norm": 1.9289932950554558,
+      "learning_rate": 9.072444444444445e-06,
+      "loss": 2.5529,
+      "step": 9175
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 2.905671046574134,
+      "learning_rate": 9.066888888888889e-06,
+      "loss": 2.5551,
+      "step": 9200
+    },
+    {
+      "epoch": 0.184,
+      "eval_loss": 2.558293342590332,
+      "eval_runtime": 42.216,
+      "eval_samples_per_second": 2.464,
+      "eval_steps_per_second": 1.232,
+      "step": 9200
+    },
+    {
+      "epoch": 0.1845,
+      "grad_norm": 2.8062526156064522,
+      "learning_rate": 9.061333333333334e-06,
+      "loss": 2.5438,
+      "step": 9225
+    },
+    {
+      "epoch": 0.185,
+      "grad_norm": 2.543328123273362,
+      "learning_rate": 9.05577777777778e-06,
+      "loss": 2.5476,
+      "step": 9250
+    },
+    {
+      "epoch": 0.1855,
+      "grad_norm": 2.396296044779414,
+      "learning_rate": 9.050222222222223e-06,
+      "loss": 2.5437,
+      "step": 9275
+    },
+    {
+      "epoch": 0.186,
+      "grad_norm": 1.980055565462775,
+      "learning_rate": 9.044666666666667e-06,
+      "loss": 2.5552,
+      "step": 9300
+    },
+    {
+      "epoch": 0.186,
+      "eval_loss": 2.557692289352417,
+      "eval_runtime": 42.6636,
+      "eval_samples_per_second": 2.438,
+      "eval_steps_per_second": 1.219,
+      "step": 9300
+    },
+    {
+      "epoch": 0.1865,
+      "grad_norm": 2.028891972183573,
+      "learning_rate": 9.039111111111112e-06,
+      "loss": 2.5603,
+      "step": 9325
+    },
+    {
+      "epoch": 0.187,
+      "grad_norm": 2.244801606614392,
+      "learning_rate": 9.033555555555557e-06,
+      "loss": 2.5565,
+      "step": 9350
+    },
+    {
+      "epoch": 0.1875,
+      "grad_norm": 2.6445168963619348,
+      "learning_rate": 9.028e-06,
+      "loss": 2.5453,
+      "step": 9375
+    },
+    {
+      "epoch": 0.188,
+      "grad_norm": 2.2015819629656543,
+      "learning_rate": 9.022444444444444e-06,
+      "loss": 2.5463,
+      "step": 9400
+    },
+    {
+      "epoch": 0.188,
+      "eval_loss": 2.555739164352417,
+      "eval_runtime": 44.4913,
+      "eval_samples_per_second": 2.338,
+      "eval_steps_per_second": 1.169,
+      "step": 9400
+    },
+    {
+      "epoch": 0.1885,
+      "grad_norm": 2.0871782907981076,
+      "learning_rate": 9.01688888888889e-06,
+      "loss": 2.5494,
+      "step": 9425
+    },
+    {
+      "epoch": 0.189,
+      "grad_norm": 2.3339796044543006,
+      "learning_rate": 9.011333333333335e-06,
+      "loss": 2.562,
+      "step": 9450
+    },
+    {
+      "epoch": 0.1895,
+      "grad_norm": 2.5447600145368257,
+      "learning_rate": 9.005777777777778e-06,
+      "loss": 2.5613,
+      "step": 9475
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.2530767222642805,
+      "learning_rate": 9.000222222222222e-06,
+      "loss": 2.5561,
+      "step": 9500
+    },
+    {
+      "epoch": 0.19,
+      "eval_loss": 2.555588960647583,
+      "eval_runtime": 42.3312,
+      "eval_samples_per_second": 2.457,
+      "eval_steps_per_second": 1.228,
+      "step": 9500
     }
   ],
   "logging_steps": 25,
@@ -3267,7 +3447,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.0197974124879938e+19,
+  "total_flos": 2.1320083808983187e+19,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null