Training checkpoint at step 12000

trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 11500,
-  "best_metric": 2.541316032409668,
-  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-11500",
-  "epoch": 0.23,
+  "best_global_step": 12000,
+  "best_metric": 2.538311243057251,
+  "best_model_checkpoint": "../runs/karpathy/fineweb-edu-100b-shuffle/google/gemma-3-1b-it/linear_adamw_wd1e-03_8x1024_mem32_bs64_hf_armt_dmem64/run_34/checkpoint-12000",
+  "epoch": 0.24,
   "eval_steps": 100,
-  "global_step": 11500,
+  "global_step": 12000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4148,6 +4148,186 @@
       "eval_samples_per_second": 2.464,
       "eval_steps_per_second": 1.232,
       "step": 11500
+    },
+    {
+      "epoch": 0.2305,
+      "grad_norm": 2.155444422697904,
+      "learning_rate": 8.550222222222223e-06,
+      "loss": 2.543,
+      "step": 11525
+    },
+    {
+      "epoch": 0.231,
+      "grad_norm": 2.5216609928964706,
+      "learning_rate": 8.544666666666668e-06,
+      "loss": 2.5339,
+      "step": 11550
+    },
+    {
+      "epoch": 0.2315,
+      "grad_norm": 3.2141643729123826,
+      "learning_rate": 8.539111111111112e-06,
+      "loss": 2.5311,
+      "step": 11575
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 2.779033714093245,
+      "learning_rate": 8.533555555555557e-06,
+      "loss": 2.5367,
+      "step": 11600
+    },
+    {
+      "epoch": 0.232,
+      "eval_loss": 2.539663553237915,
+      "eval_runtime": 42.1104,
+      "eval_samples_per_second": 2.47,
+      "eval_steps_per_second": 1.235,
+      "step": 11600
+    },
+    {
+      "epoch": 0.2325,
+      "grad_norm": 2.0599049344871134,
+      "learning_rate": 8.528e-06,
+      "loss": 2.5406,
+      "step": 11625
+    },
+    {
+      "epoch": 0.233,
+      "grad_norm": 2.1617162796171536,
+      "learning_rate": 8.522444444444446e-06,
+      "loss": 2.5244,
+      "step": 11650
+    },
+    {
+      "epoch": 0.2335,
+      "grad_norm": 2.4286224889340926,
+      "learning_rate": 8.51688888888889e-06,
+      "loss": 2.5364,
+      "step": 11675
+    },
+    {
+      "epoch": 0.234,
+      "grad_norm": 2.0435359432545424,
+      "learning_rate": 8.511333333333334e-06,
+      "loss": 2.5332,
+      "step": 11700
+    },
+    {
+      "epoch": 0.234,
+      "eval_loss": 2.539963960647583,
+      "eval_runtime": 42.1502,
+      "eval_samples_per_second": 2.467,
+      "eval_steps_per_second": 1.234,
+      "step": 11700
+    },
+    {
+      "epoch": 0.2345,
+      "grad_norm": 2.6031764141012195,
+      "learning_rate": 8.505777777777778e-06,
+      "loss": 2.5292,
+      "step": 11725
+    },
+    {
+      "epoch": 0.235,
+      "grad_norm": 2.2484621657042427,
+      "learning_rate": 8.500222222222223e-06,
+      "loss": 2.523,
+      "step": 11750
+    },
+    {
+      "epoch": 0.2355,
+      "grad_norm": 2.854177673999505,
+      "learning_rate": 8.494666666666668e-06,
+      "loss": 2.5218,
+      "step": 11775
+    },
+    {
+      "epoch": 0.236,
+      "grad_norm": 2.0770100967771055,
+      "learning_rate": 8.489111111111112e-06,
+      "loss": 2.534,
+      "step": 11800
+    },
+    {
+      "epoch": 0.236,
+      "eval_loss": 2.538536548614502,
+      "eval_runtime": 42.3875,
+      "eval_samples_per_second": 2.454,
+      "eval_steps_per_second": 1.227,
+      "step": 11800
+    },
+    {
+      "epoch": 0.2365,
+      "grad_norm": 2.391823444522325,
+      "learning_rate": 8.483555555555556e-06,
+      "loss": 2.5211,
+      "step": 11825
+    },
+    {
+      "epoch": 0.237,
+      "grad_norm": 2.333238897849914,
+      "learning_rate": 8.478e-06,
+      "loss": 2.5238,
+      "step": 11850
+    },
+    {
+      "epoch": 0.2375,
+      "grad_norm": 2.1636671466235256,
+      "learning_rate": 8.472444444444446e-06,
+      "loss": 2.5378,
+      "step": 11875
+    },
+    {
+      "epoch": 0.238,
+      "grad_norm": 2.5877564973697607,
+      "learning_rate": 8.46688888888889e-06,
+      "loss": 2.5415,
+      "step": 11900
+    },
+    {
+      "epoch": 0.238,
+      "eval_loss": 2.538837194442749,
+      "eval_runtime": 42.2059,
+      "eval_samples_per_second": 2.464,
+      "eval_steps_per_second": 1.232,
+      "step": 11900
+    },
+    {
+      "epoch": 0.2385,
+      "grad_norm": 2.1416643296031785,
+      "learning_rate": 8.461333333333333e-06,
+      "loss": 2.525,
+      "step": 11925
+    },
+    {
+      "epoch": 0.239,
+      "grad_norm": 2.213813959028046,
+      "learning_rate": 8.455777777777778e-06,
+      "loss": 2.5416,
+      "step": 11950
+    },
+    {
+      "epoch": 0.2395,
+      "grad_norm": 2.759854381361929,
+      "learning_rate": 8.450222222222224e-06,
+      "loss": 2.5355,
+      "step": 11975
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 2.050520488248713,
+      "learning_rate": 8.444666666666667e-06,
+      "loss": 2.5263,
+      "step": 12000
+    },
+    {
+      "epoch": 0.24,
+      "eval_loss": 2.538311243057251,
+      "eval_runtime": 42.2256,
+      "eval_samples_per_second": 2.463,
+      "eval_steps_per_second": 1.231,
+      "step": 12000
     }
   ],
   "logging_steps": 25,
@@ -4167,7 +4347,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.5808522545396187e+19,
+  "total_flos": 2.6930632229499437e+19,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null