Training in progress, step 14500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8aeb85e50392772e4b771ed01067db7dd33a6869f84c04ae7432a0dd055a0f19
+oid sha256:a3576f2655b1ba80d212588b793a4ccc62cae448fb8536ce80c2cb8519f9e8da
 size 12017472

last-checkpoint/global_step14500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8ddbbfe6677bfeb3dea29b30df97965b929938a7c03ad9eacba0e52ef12377
+size 71982309

last-checkpoint/global_step14500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c8de513350d3a396702450256a3434f4f6d8424161c0019906936c1e1f1caa3
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step14000
+global_step14500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f95af1290403efd8633702ca95f724b8eeb1c11b90d76b5a45554aca28009c5
+oid sha256:e877aa0d3a3d9a4fe852642f23daa221d76931700b5fdfe8ba4090a8a19bcbbb
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 13750,
-  "best_metric": 0.5387488603591919,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-13500",
-  "epoch": 10.174513724777313,
+  "best_global_step": 14500,
+  "best_metric": 0.5384897589683533,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-14500",
+  "epoch": 10.538083984730049,
   "eval_steps": 250,
-  "global_step": 14000,
+  "global_step": 14500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5617,6 +5617,206 @@
       "eval_samples_per_second": 43.639,
       "eval_steps_per_second": 5.462,
       "step": 14000
+    },
+    {
+      "epoch": 10.19269223777495,
+      "grad_norm": 0.8508243560791016,
+      "learning_rate": 4.4477516452882655e-06,
+      "loss": 0.5064,
+      "mean_token_accuracy": 0.8425439709424972,
+      "num_tokens": 308806902.0,
+      "step": 14025
+    },
+    {
+      "epoch": 10.210870750772587,
+      "grad_norm": 0.8245001435279846,
+      "learning_rate": 4.360434283160126e-06,
+      "loss": 0.5089,
+      "mean_token_accuracy": 0.8431083789467811,
+      "num_tokens": 309352182.0,
+      "step": 14050
+    },
+    {
+      "epoch": 10.229049263770223,
+      "grad_norm": 0.8090792298316956,
+      "learning_rate": 4.273933105490162e-06,
+      "loss": 0.5123,
+      "mean_token_accuracy": 0.8400397875905037,
+      "num_tokens": 309919307.0,
+      "step": 14075
+    },
+    {
+      "epoch": 10.247227776767861,
+      "grad_norm": 0.9191139936447144,
+      "learning_rate": 4.188250093248547e-06,
+      "loss": 0.5021,
+      "mean_token_accuracy": 0.8438076037168503,
+      "num_tokens": 310468181.0,
+      "step": 14100
+    },
+    {
+      "epoch": 10.265406289765497,
+      "grad_norm": 0.8430826663970947,
+      "learning_rate": 4.103387208668594e-06,
+      "loss": 0.5103,
+      "mean_token_accuracy": 0.8410224625468254,
+      "num_tokens": 311012563.0,
+      "step": 14125
+    },
+    {
+      "epoch": 10.283584802763134,
+      "grad_norm": 0.8337134122848511,
+      "learning_rate": 4.019346395201793e-06,
+      "loss": 0.5059,
+      "mean_token_accuracy": 0.8416058418154716,
+      "num_tokens": 311558333.0,
+      "step": 14150
+    },
+    {
+      "epoch": 10.301763315760772,
+      "grad_norm": 0.8520947694778442,
+      "learning_rate": 3.936129577473344e-06,
+      "loss": 0.5117,
+      "mean_token_accuracy": 0.839869918525219,
+      "num_tokens": 312128294.0,
+      "step": 14175
+    },
+    {
+      "epoch": 10.319941828758408,
+      "grad_norm": 0.8563548922538757,
+      "learning_rate": 3.853738661238042e-06,
+      "loss": 0.5162,
+      "mean_token_accuracy": 0.8394653937220573,
+      "num_tokens": 312689462.0,
+      "step": 14200
+    },
+    {
+      "epoch": 10.338120341756044,
+      "grad_norm": 0.8299646377563477,
+      "learning_rate": 3.7721755333366326e-06,
+      "loss": 0.508,
+      "mean_token_accuracy": 0.8402037498354912,
+      "num_tokens": 313254544.0,
+      "step": 14225
+    },
+    {
+      "epoch": 10.356298854753682,
+      "grad_norm": 0.865742027759552,
+      "learning_rate": 3.691442061652657e-06,
+      "loss": 0.5106,
+      "mean_token_accuracy": 0.8408624231815338,
+      "num_tokens": 313792753.0,
+      "step": 14250
+    },
+    {
+      "epoch": 10.356298854753682,
+      "eval_loss": 0.5386558175086975,
+      "eval_mean_token_accuracy": 0.8323602951040455,
+      "eval_num_tokens": 313792753.0,
+      "eval_runtime": 111.6679,
+      "eval_samples_per_second": 43.791,
+      "eval_steps_per_second": 5.481,
+      "step": 14250
+    },
+    {
+      "epoch": 10.374477367751318,
+      "grad_norm": 0.9042721390724182,
+      "learning_rate": 3.611540095069592e-06,
+      "loss": 0.5121,
+      "mean_token_accuracy": 0.8402319389581681,
+      "num_tokens": 314338619.0,
+      "step": 14275
+    },
+    {
+      "epoch": 10.392655880748954,
+      "grad_norm": 0.9073200225830078,
+      "learning_rate": 3.5324714634285796e-06,
+      "loss": 0.5095,
+      "mean_token_accuracy": 0.8411319550871849,
+      "num_tokens": 314874371.0,
+      "step": 14300
+    },
+    {
+      "epoch": 10.410834393746592,
+      "grad_norm": 0.8187711238861084,
+      "learning_rate": 3.454237977486483e-06,
+      "loss": 0.5051,
+      "mean_token_accuracy": 0.8423356208205223,
+      "num_tokens": 315434419.0,
+      "step": 14325
+    },
+    {
+      "epoch": 10.429012906744228,
+      "grad_norm": 0.8220618963241577,
+      "learning_rate": 3.3768414288744268e-06,
+      "loss": 0.5118,
+      "mean_token_accuracy": 0.8405367460846901,
+      "num_tokens": 315967309.0,
+      "step": 14350
+    },
+    {
+      "epoch": 10.447191419741864,
+      "grad_norm": 0.9530115723609924,
+      "learning_rate": 3.3002835900567677e-06,
+      "loss": 0.5121,
+      "mean_token_accuracy": 0.8401629340648651,
+      "num_tokens": 316508469.0,
+      "step": 14375
+    },
+    {
+      "epoch": 10.465369932739502,
+      "grad_norm": 0.8760950565338135,
+      "learning_rate": 3.224566214290521e-06,
+      "loss": 0.5057,
+      "mean_token_accuracy": 0.8424499598145485,
+      "num_tokens": 317046765.0,
+      "step": 14400
+    },
+    {
+      "epoch": 10.483548445737139,
+      "grad_norm": 0.8828684091567993,
+      "learning_rate": 3.1496910355851785e-06,
+      "loss": 0.509,
+      "mean_token_accuracy": 0.841154874265194,
+      "num_tokens": 317596305.0,
+      "step": 14425
+    },
+    {
+      "epoch": 10.501726958734775,
+      "grad_norm": 0.7962938547134399,
+      "learning_rate": 3.0756597686630064e-06,
+      "loss": 0.5171,
+      "mean_token_accuracy": 0.8385607668757439,
+      "num_tokens": 318163982.0,
+      "step": 14450
+    },
+    {
+      "epoch": 10.519905471732413,
+      "grad_norm": 0.83053058385849,
+      "learning_rate": 3.0024741089197975e-06,
+      "loss": 0.508,
+      "mean_token_accuracy": 0.8415687373280525,
+      "num_tokens": 318707187.0,
+      "step": 14475
+    },
+    {
+      "epoch": 10.538083984730049,
+      "grad_norm": 0.8857102394104004,
+      "learning_rate": 2.9301357323860168e-06,
+      "loss": 0.5138,
+      "mean_token_accuracy": 0.839360601902008,
+      "num_tokens": 319249758.0,
+      "step": 14500
+    },
+    {
+      "epoch": 10.538083984730049,
+      "eval_loss": 0.5384897589683533,
+      "eval_mean_token_accuracy": 0.8324334327301948,
+      "eval_num_tokens": 319249758.0,
+      "eval_runtime": 110.9365,
+      "eval_samples_per_second": 44.079,
+      "eval_steps_per_second": 5.517,
+      "step": 14500
     }
   ],
   "logging_steps": 25,
@@ -5636,7 +5836,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.776595356229304e+17,
+  "total_flos": 8.05384191213568e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null