Training in progress, step 13500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db7100a45db008b406f2052c3128c88105424250504770688fd4dc9c99873aaa
+oid sha256:1311c9a69e5604b2001ceda10c832e98119547c0e33d82afe5989665de514c3e
 size 12017472

last-checkpoint/global_step13500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e5b7fd370b88c57ef6538390266dc426bccc73daf55376f38bfe8614c792f79
+size 71982309

last-checkpoint/global_step13500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2751dde639a5d12f58ec51183d6aef63115a33b7c76078f4a229de16b57b14e
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step13000
+global_step13500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f08b6a541827b05fd5c665552fdb001f91a00f8dfca00dd95e706aac683d501
+oid sha256:76d48473cf121167cd401e1842d406e7e5686b60208f0336b7552832934ccc04
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 13000,
-  "best_metric": 0.540317952632904,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-13000",
-  "epoch": 9.44791856026177,
+  "best_global_step": 13500,
+  "best_metric": 0.5390045046806335,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-13500",
+  "epoch": 9.811488820214507,
   "eval_steps": 250,
-  "global_step": 13000,
+  "global_step": 13500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5217,6 +5217,206 @@
       "eval_samples_per_second": 42.989,
       "eval_steps_per_second": 5.38,
       "step": 13000
+    },
+    {
+      "epoch": 9.466097073259407,
+      "grad_norm": 0.8422950506210327,
+      "learning_rate": 8.584715779344832e-06,
+      "loss": 0.5187,
+      "mean_token_accuracy": 0.8383197170495987,
+      "num_tokens": 286863014.0,
+      "step": 13025
+    },
+    {
+      "epoch": 9.484275586257045,
+      "grad_norm": 0.7978519797325134,
+      "learning_rate": 8.466584529700203e-06,
+      "loss": 0.515,
+      "mean_token_accuracy": 0.839700258076191,
+      "num_tokens": 287406511.0,
+      "step": 13050
+    },
+    {
+      "epoch": 9.502454099254681,
+      "grad_norm": 0.8645240664482117,
+      "learning_rate": 8.349175429248554e-06,
+      "loss": 0.5238,
+      "mean_token_accuracy": 0.8366273155808449,
+      "num_tokens": 287962024.0,
+      "step": 13075
+    },
+    {
+      "epoch": 9.520632612252317,
+      "grad_norm": 0.8597573041915894,
+      "learning_rate": 8.232491166784782e-06,
+      "loss": 0.5159,
+      "mean_token_accuracy": 0.8379004463553429,
+      "num_tokens": 288527560.0,
+      "step": 13100
+    },
+    {
+      "epoch": 9.538811125249955,
+      "grad_norm": 0.8828545808792114,
+      "learning_rate": 8.116534414504232e-06,
+      "loss": 0.5118,
+      "mean_token_accuracy": 0.8406583109498024,
+      "num_tokens": 289060843.0,
+      "step": 13125
+    },
+    {
+      "epoch": 9.556989638247591,
+      "grad_norm": 0.8724490404129028,
+      "learning_rate": 8.00130782794148e-06,
+      "loss": 0.5239,
+      "mean_token_accuracy": 0.8369137379527092,
+      "num_tokens": 289603965.0,
+      "step": 13150
+    },
+    {
+      "epoch": 9.575168151245228,
+      "grad_norm": 0.8818336129188538,
+      "learning_rate": 7.886814045909515e-06,
+      "loss": 0.5244,
+      "mean_token_accuracy": 0.8372589892148972,
+      "num_tokens": 290146905.0,
+      "step": 13175
+    },
+    {
+      "epoch": 9.593346664242866,
+      "grad_norm": 0.9488387703895569,
+      "learning_rate": 7.773055690439326e-06,
+      "loss": 0.5131,
+      "mean_token_accuracy": 0.8400958624482154,
+      "num_tokens": 290702107.0,
+      "step": 13200
+    },
+    {
+      "epoch": 9.611525177240502,
+      "grad_norm": 0.8438289165496826,
+      "learning_rate": 7.66003536671982e-06,
+      "loss": 0.5131,
+      "mean_token_accuracy": 0.8400224041938782,
+      "num_tokens": 291241779.0,
+      "step": 13225
+    },
+    {
+      "epoch": 9.629703690238138,
+      "grad_norm": 0.8664806485176086,
+      "learning_rate": 7.547755663038212e-06,
+      "loss": 0.5107,
+      "mean_token_accuracy": 0.8407774633169174,
+      "num_tokens": 291796633.0,
+      "step": 13250
+    },
+    {
+      "epoch": 9.629703690238138,
+      "eval_loss": 0.5401590466499329,
+      "eval_mean_token_accuracy": 0.8320043968414169,
+      "eval_num_tokens": 291796633.0,
+      "eval_runtime": 112.5867,
+      "eval_samples_per_second": 43.433,
+      "eval_steps_per_second": 5.436,
+      "step": 13250
+    },
+    {
+      "epoch": 9.647882203235776,
+      "grad_norm": 0.8282386064529419,
+      "learning_rate": 7.436219150720698e-06,
+      "loss": 0.5155,
+      "mean_token_accuracy": 0.84046880453825,
+      "num_tokens": 292340922.0,
+      "step": 13275
+    },
+    {
+      "epoch": 9.666060716233412,
+      "grad_norm": 0.872983455657959,
+      "learning_rate": 7.325428384073592e-06,
+      "loss": 0.5231,
+      "mean_token_accuracy": 0.8363588589429856,
+      "num_tokens": 292895625.0,
+      "step": 13300
+    },
+    {
+      "epoch": 9.684239229231048,
+      "grad_norm": 0.8708329200744629,
+      "learning_rate": 7.215385900324832e-06,
+      "loss": 0.5144,
+      "mean_token_accuracy": 0.8397229793667793,
+      "num_tokens": 293448542.0,
+      "step": 13325
+    },
+    {
+      "epoch": 9.702417742228686,
+      "grad_norm": 0.8467702269554138,
+      "learning_rate": 7.106094219565869e-06,
+      "loss": 0.5171,
+      "mean_token_accuracy": 0.8385615301132202,
+      "num_tokens": 294000478.0,
+      "step": 13350
+    },
+    {
+      "epoch": 9.720596255226322,
+      "grad_norm": 0.8231089115142822,
+      "learning_rate": 6.9975558446939665e-06,
+      "loss": 0.5132,
+      "mean_token_accuracy": 0.8399266812205315,
+      "num_tokens": 294557047.0,
+      "step": 13375
+    },
+    {
+      "epoch": 9.738774768223958,
+      "grad_norm": 0.9206160306930542,
+      "learning_rate": 6.8897732613548526e-06,
+      "loss": 0.5096,
+      "mean_token_accuracy": 0.8407321670651435,
+      "num_tokens": 295104353.0,
+      "step": 13400
+    },
+    {
+      "epoch": 9.756953281221596,
+      "grad_norm": 0.8946228623390198,
+      "learning_rate": 6.782748937885842e-06,
+      "loss": 0.5157,
+      "mean_token_accuracy": 0.8397801405191422,
+      "num_tokens": 295655574.0,
+      "step": 13425
+    },
+    {
+      "epoch": 9.775131794219233,
+      "grad_norm": 0.7474434971809387,
+      "learning_rate": 6.6764853252592585e-06,
+      "loss": 0.5217,
+      "mean_token_accuracy": 0.8362213695049285,
+      "num_tokens": 296223611.0,
+      "step": 13450
+    },
+    {
+      "epoch": 9.79331030721687,
+      "grad_norm": 0.8649734258651733,
+      "learning_rate": 6.5709848570263324e-06,
+      "loss": 0.5151,
+      "mean_token_accuracy": 0.838211068212986,
+      "num_tokens": 296787088.0,
+      "step": 13475
+    },
+    {
+      "epoch": 9.811488820214507,
+      "grad_norm": 0.7948579788208008,
+      "learning_rate": 6.466249949261474e-06,
+      "loss": 0.5165,
+      "mean_token_accuracy": 0.8387623742222786,
+      "num_tokens": 297344033.0,
+      "step": 13500
+    },
+    {
+      "epoch": 9.811488820214507,
+      "eval_loss": 0.5390045046806335,
+      "eval_mean_token_accuracy": 0.8321733054966708,
+      "eval_num_tokens": 297344033.0,
+      "eval_runtime": 113.1601,
+      "eval_samples_per_second": 43.213,
+      "eval_steps_per_second": 5.408,
+      "step": 13500
     }
   ],
   "logging_steps": 25,
@@ -5236,7 +5436,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.220988366123172e+17,
+  "total_flos": 7.499312044798116e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null