Training checkpoint at step 13000

trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 12400,
-  "best_metric": 2.535456657409668,
-  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-12000",
-  "epoch": 0.25,
+  "best_global_step": 13000,
+  "best_metric": 2.532376766204834,
+  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-13000",
+  "epoch": 0.26,
   "eval_steps": 100,
-  "global_step": 12500,
+  "global_step": 13000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4508,6 +4508,186 @@
       "eval_samples_per_second": 2.473,
       "eval_steps_per_second": 1.237,
       "step": 12500
+    },
+    {
+      "epoch": 0.2505,
+      "grad_norm": 2.019140906115923,
+      "learning_rate": 8.328e-06,
+      "loss": 2.5241,
+      "step": 12525
+    },
+    {
+      "epoch": 0.251,
+      "grad_norm": 1.9012303831260067,
+      "learning_rate": 8.322444444444446e-06,
+      "loss": 2.5354,
+      "step": 12550
+    },
+    {
+      "epoch": 0.2515,
+      "grad_norm": 1.7607101331370496,
+      "learning_rate": 8.31688888888889e-06,
+      "loss": 2.5254,
+      "step": 12575
+    },
+    {
+      "epoch": 0.252,
+      "grad_norm": 2.5505055208286933,
+      "learning_rate": 8.311333333333333e-06,
+      "loss": 2.5294,
+      "step": 12600
+    },
+    {
+      "epoch": 0.252,
+      "eval_loss": 2.535231351852417,
+      "eval_runtime": 41.9731,
+      "eval_samples_per_second": 2.478,
+      "eval_steps_per_second": 1.239,
+      "step": 12600
+    },
+    {
+      "epoch": 0.2525,
+      "grad_norm": 1.6218420390627293,
+      "learning_rate": 8.305777777777778e-06,
+      "loss": 2.5262,
+      "step": 12625
+    },
+    {
+      "epoch": 0.253,
+      "grad_norm": 2.0991897222525115,
+      "learning_rate": 8.300222222222223e-06,
+      "loss": 2.5206,
+      "step": 12650
+    },
+    {
+      "epoch": 0.2535,
+      "grad_norm": 2.478785246720621,
+      "learning_rate": 8.294666666666667e-06,
+      "loss": 2.5275,
+      "step": 12675
+    },
+    {
+      "epoch": 0.254,
+      "grad_norm": 2.141371973093057,
+      "learning_rate": 8.289111111111112e-06,
+      "loss": 2.5323,
+      "step": 12700
+    },
+    {
+      "epoch": 0.254,
+      "eval_loss": 2.5341796875,
+      "eval_runtime": 42.2622,
+      "eval_samples_per_second": 2.461,
+      "eval_steps_per_second": 1.23,
+      "step": 12700
+    },
+    {
+      "epoch": 0.2545,
+      "grad_norm": 2.269733740633448,
+      "learning_rate": 8.283555555555556e-06,
+      "loss": 2.5367,
+      "step": 12725
+    },
+    {
+      "epoch": 0.255,
+      "grad_norm": 1.893617133257015,
+      "learning_rate": 8.278000000000001e-06,
+      "loss": 2.5257,
+      "step": 12750
+    },
+    {
+      "epoch": 0.2555,
+      "grad_norm": 1.751381032940087,
+      "learning_rate": 8.272444444444445e-06,
+      "loss": 2.5276,
+      "step": 12775
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 2.6264391487699545,
+      "learning_rate": 8.26688888888889e-06,
+      "loss": 2.5281,
+      "step": 12800
+    },
+    {
+      "epoch": 0.256,
+      "eval_loss": 2.534780740737915,
+      "eval_runtime": 42.0037,
+      "eval_samples_per_second": 2.476,
+      "eval_steps_per_second": 1.238,
+      "step": 12800
+    },
+    {
+      "epoch": 0.2565,
+      "grad_norm": 2.9544216590918766,
+      "learning_rate": 8.261333333333335e-06,
+      "loss": 2.5159,
+      "step": 12825
+    },
+    {
+      "epoch": 0.257,
+      "grad_norm": 1.703574826031134,
+      "learning_rate": 8.255777777777779e-06,
+      "loss": 2.5314,
+      "step": 12850
+    },
+    {
+      "epoch": 0.2575,
+      "grad_norm": 2.23456733038464,
+      "learning_rate": 8.250222222222222e-06,
+      "loss": 2.5301,
+      "step": 12875
+    },
+    {
+      "epoch": 0.258,
+      "grad_norm": 2.0236952351089132,
+      "learning_rate": 8.244666666666667e-06,
+      "loss": 2.5274,
+      "step": 12900
+    },
+    {
+      "epoch": 0.258,
+      "eval_loss": 2.532827615737915,
+      "eval_runtime": 42.2742,
+      "eval_samples_per_second": 2.46,
+      "eval_steps_per_second": 1.23,
+      "step": 12900
+    },
+    {
+      "epoch": 0.2585,
+      "grad_norm": 1.9175658573019432,
+      "learning_rate": 8.239111111111113e-06,
+      "loss": 2.5293,
+      "step": 12925
+    },
+    {
+      "epoch": 0.259,
+      "grad_norm": 2.227745372848629,
+      "learning_rate": 8.233555555555556e-06,
+      "loss": 2.5346,
+      "step": 12950
+    },
+    {
+      "epoch": 0.2595,
+      "grad_norm": 2.0320264112024375,
+      "learning_rate": 8.228e-06,
+      "loss": 2.5133,
+      "step": 12975
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 2.3254627331546636,
+      "learning_rate": 8.222444444444445e-06,
+      "loss": 2.5257,
+      "step": 13000
+    },
+    {
+      "epoch": 0.26,
+      "eval_loss": 2.532376766204834,
+      "eval_runtime": 42.0555,
+      "eval_samples_per_second": 2.473,
+      "eval_steps_per_second": 1.236,
+      "step": 13000
     }
   ],
   "logging_steps": 25,
@@ -4527,7 +4707,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.8052741913602687e+19,
+  "total_flos": 2.9174851597705937e+19,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null