Training in progress, step 11500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12a26c5ae72d4aef0518c84efdc9e2d761d59aa9a43290c6e5e8c3c753145a81
+oid sha256:d739a46be07afc08058bcee6abb1772a84e044deaf39817666f3049bcf653c23
 size 12017472

last-checkpoint/global_step11500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d600239e729c4ded64651931f6cb684445e13a28cc9c8180766ac66dd15525f4
+size 71982309

last-checkpoint/global_step11500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f2820de752ec7bbd3367caf1e9cd13773cad956bad3832cf6ffd7af2da666c1
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step11000
+global_step11500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5874e4cafeab6bbc56f1a9ea496d34e4351f44b1e8ee07a759c92cdeb51ddda5
+oid sha256:d17e6956d333adf450e550fb2bbfe82bc47be67acb5350845a13faa81c890b40
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 11000,
-  "best_metric": 0.5450185537338257,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-11000",
-  "epoch": 7.9947282312306855,
+  "best_global_step": 11500,
+  "best_metric": 0.544745683670044,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-11500",
+  "epoch": 8.357753135793493,
   "eval_steps": 250,
-  "global_step": 11000,
+  "global_step": 11500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4417,6 +4417,206 @@
       "eval_samples_per_second": 43.083,
       "eval_steps_per_second": 5.392,
       "step": 11000
+    },
+    {
+      "epoch": 8.012361388838393,
+      "grad_norm": 0.8800711035728455,
+      "learning_rate": 2.0108403898867298e-05,
+      "loss": 0.511,
+      "mean_token_accuracy": 0.8405692936833372,
+      "num_tokens": 242787188.0,
+      "step": 11025
+    },
+    {
+      "epoch": 8.030539901836029,
+      "grad_norm": 0.8134773969650269,
+      "learning_rate": 1.994255888710432e-05,
+      "loss": 0.5161,
+      "mean_token_accuracy": 0.8396863287687302,
+      "num_tokens": 243323359.0,
+      "step": 11050
+    },
+    {
+      "epoch": 8.048718414833667,
+      "grad_norm": 0.8506718277931213,
+      "learning_rate": 1.977717321235564e-05,
+      "loss": 0.5184,
+      "mean_token_accuracy": 0.8389563143253327,
+      "num_tokens": 243872329.0,
+      "step": 11075
+    },
+    {
+      "epoch": 8.066896927831303,
+      "grad_norm": 0.8440150022506714,
+      "learning_rate": 1.9612250662131406e-05,
+      "loss": 0.5154,
+      "mean_token_accuracy": 0.8382254421710968,
+      "num_tokens": 244414078.0,
+      "step": 11100
+    },
+    {
+      "epoch": 8.08507544082894,
+      "grad_norm": 0.8506153225898743,
+      "learning_rate": 1.9447795013335734e-05,
+      "loss": 0.5232,
+      "mean_token_accuracy": 0.8369153061509133,
+      "num_tokens": 244971752.0,
+      "step": 11125
+    },
+    {
+      "epoch": 8.103253953826577,
+      "grad_norm": 0.8672150373458862,
+      "learning_rate": 1.9283810032180205e-05,
+      "loss": 0.512,
+      "mean_token_accuracy": 0.8409202411770821,
+      "num_tokens": 245520458.0,
+      "step": 11150
+    },
+    {
+      "epoch": 8.121432466824213,
+      "grad_norm": 0.8128538727760315,
+      "learning_rate": 1.9120299474097583e-05,
+      "loss": 0.5209,
+      "mean_token_accuracy": 0.837473659813404,
+      "num_tokens": 246088615.0,
+      "step": 11175
+    },
+    {
+      "epoch": 8.139610979821851,
+      "grad_norm": 0.7617402076721191,
+      "learning_rate": 1.8957267083655835e-05,
+      "loss": 0.5153,
+      "mean_token_accuracy": 0.8394093406200409,
+      "num_tokens": 246630403.0,
+      "step": 11200
+    },
+    {
+      "epoch": 8.157789492819488,
+      "grad_norm": 0.8872790932655334,
+      "learning_rate": 1.8794716594472376e-05,
+      "loss": 0.5179,
+      "mean_token_accuracy": 0.838570873439312,
+      "num_tokens": 247175258.0,
+      "step": 11225
+    },
+    {
+      "epoch": 8.175968005817124,
+      "grad_norm": 0.7966537475585938,
+      "learning_rate": 1.8632651729128564e-05,
+      "loss": 0.5209,
+      "mean_token_accuracy": 0.8365825054049492,
+      "num_tokens": 247743507.0,
+      "step": 11250
+    },
+    {
+      "epoch": 8.175968005817124,
+      "eval_loss": 0.5448639392852783,
+      "eval_mean_token_accuracy": 0.8303035672973184,
+      "eval_num_tokens": 247743507.0,
+      "eval_runtime": 113.7219,
+      "eval_samples_per_second": 43.0,
+      "eval_steps_per_second": 5.382,
+      "step": 11250
+    },
+    {
+      "epoch": 8.194146518814762,
+      "grad_norm": 0.9003967642784119,
+      "learning_rate": 1.847107619908445e-05,
+      "loss": 0.5157,
+      "mean_token_accuracy": 0.8391850134730339,
+      "num_tokens": 248275961.0,
+      "step": 11275
+    },
+    {
+      "epoch": 8.212325031812398,
+      "grad_norm": 0.7678829431533813,
+      "learning_rate": 1.8309993704593756e-05,
+      "loss": 0.5175,
+      "mean_token_accuracy": 0.8387827044725418,
+      "num_tokens": 248835571.0,
+      "step": 11300
+    },
+    {
+      "epoch": 8.230503544810034,
+      "grad_norm": 0.8297247290611267,
+      "learning_rate": 1.8149407934619215e-05,
+      "loss": 0.5213,
+      "mean_token_accuracy": 0.8382138457894325,
+      "num_tokens": 249386561.0,
+      "step": 11325
+    },
+    {
+      "epoch": 8.248682057807672,
+      "grad_norm": 0.8659992218017578,
+      "learning_rate": 1.798932256674798e-05,
+      "loss": 0.5181,
+      "mean_token_accuracy": 0.8384436306357383,
+      "num_tokens": 249964812.0,
+      "step": 11350
+    },
+    {
+      "epoch": 8.266860570805308,
+      "grad_norm": 0.8487904071807861,
+      "learning_rate": 1.782974126710748e-05,
+      "loss": 0.5243,
+      "mean_token_accuracy": 0.8366836148500443,
+      "num_tokens": 250524273.0,
+      "step": 11375
+    },
+    {
+      "epoch": 8.285039083802944,
+      "grad_norm": 0.8609278202056885,
+      "learning_rate": 1.767066769028143e-05,
+      "loss": 0.521,
+      "mean_token_accuracy": 0.8375069627165794,
+      "num_tokens": 251087296.0,
+      "step": 11400
+    },
+    {
+      "epoch": 8.303217596800582,
+      "grad_norm": 0.8295932412147522,
+      "learning_rate": 1.7512105479226144e-05,
+      "loss": 0.5205,
+      "mean_token_accuracy": 0.8369895967841149,
+      "num_tokens": 251637750.0,
+      "step": 11425
+    },
+    {
+      "epoch": 8.321396109798219,
+      "grad_norm": 0.8182777166366577,
+      "learning_rate": 1.7354058265187116e-05,
+      "loss": 0.5224,
+      "mean_token_accuracy": 0.8378088471293449,
+      "num_tokens": 252191575.0,
+      "step": 11450
+    },
+    {
+      "epoch": 8.339574622795855,
+      "grad_norm": 0.7869584560394287,
+      "learning_rate": 1.7196529667615838e-05,
+      "loss": 0.518,
+      "mean_token_accuracy": 0.8380302327871323,
+      "num_tokens": 252747951.0,
+      "step": 11475
+    },
+    {
+      "epoch": 8.357753135793493,
+      "grad_norm": 0.7901642918586731,
+      "learning_rate": 1.7039523294086968e-05,
+      "loss": 0.5188,
+      "mean_token_accuracy": 0.8381170380115509,
+      "num_tokens": 253308593.0,
+      "step": 11500
+    },
+    {
+      "epoch": 8.357753135793493,
+      "eval_loss": 0.544745683670044,
+      "eval_mean_token_accuracy": 0.8305268148386401,
+      "eval_num_tokens": 253308593.0,
+      "eval_runtime": 114.0858,
+      "eval_samples_per_second": 42.862,
+      "eval_steps_per_second": 5.364,
+      "step": 11500
     }
   ],
   "logging_steps": 25,
@@ -4436,7 +4636,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.109487891459604e+17,
+  "total_flos": 6.387220694035333e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null