Training in progress, step 8500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07328eeab3354e923721dcf5dc81f90b0d35763e3d2da61af29191c7a2e7c269
+oid sha256:9ebf61e4d96852d448117769c201ad05ba342a330472e0c4f7a17e11064f0353
 size 12017472

last-checkpoint/global_step8500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9faf9cbf90034bc7c6932f8e7206842c828fa7da1b8607e63186a879d9056a1
+size 71982309

last-checkpoint/global_step8500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6b067642615c61b221d7f18c7372bb6f436f7c58261bf5c3784fb34aee8621a
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step8000
+global_step8500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63794b157bba20e542419c9f34c95871186573432daaa310cf18fc7cd73ac609
+oid sha256:b568051b69d3c00915acebc4a453fb6368ef5a43d2d24e5d733c17ec637f2069
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 8000,
-  "best_metric": 0.5604261755943298,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-8000",
-  "epoch": 5.814397382294128,
+  "best_global_step": 8500,
+  "best_metric": 0.5578206181526184,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-8500",
+  "epoch": 6.1774222868569355,
   "eval_steps": 250,
-  "global_step": 8000,
+  "global_step": 8500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3217,6 +3217,206 @@
       "eval_samples_per_second": 43.235,
       "eval_steps_per_second": 5.411,
       "step": 8000
+    },
+    {
+      "epoch": 5.8325758952917655,
+      "grad_norm": 0.774117648601532,
+      "learning_rate": 4.215052865315844e-05,
+      "loss": 0.5424,
+      "mean_token_accuracy": 0.8316737848520279,
+      "num_tokens": 176748107.0,
+      "step": 8025
+    },
+    {
+      "epoch": 5.850754408289402,
+      "grad_norm": 0.7777197360992432,
+      "learning_rate": 4.1959361143150435e-05,
+      "loss": 0.5522,
+      "mean_token_accuracy": 0.8278136014938354,
+      "num_tokens": 177303487.0,
+      "step": 8050
+    },
+    {
+      "epoch": 5.868932921287039,
+      "grad_norm": 0.7735779285430908,
+      "learning_rate": 4.176814876166096e-05,
+      "loss": 0.5499,
+      "mean_token_accuracy": 0.8293804588913918,
+      "num_tokens": 177849190.0,
+      "step": 8075
+    },
+    {
+      "epoch": 5.887111434284676,
+      "grad_norm": 0.8152751922607422,
+      "learning_rate": 4.157689588765956e-05,
+      "loss": 0.5504,
+      "mean_token_accuracy": 0.8286561304330826,
+      "num_tokens": 178400389.0,
+      "step": 8100
+    },
+    {
+      "epoch": 5.905289947282312,
+      "grad_norm": 0.7434157729148865,
+      "learning_rate": 4.138560690104317e-05,
+      "loss": 0.5603,
+      "mean_token_accuracy": 0.8263833120465278,
+      "num_tokens": 178953283.0,
+      "step": 8125
+    },
+    {
+      "epoch": 5.923468460279949,
+      "grad_norm": 0.7802156805992126,
+      "learning_rate": 4.119428618253569e-05,
+      "loss": 0.5476,
+      "mean_token_accuracy": 0.8301771306991577,
+      "num_tokens": 179508280.0,
+      "step": 8150
+    },
+    {
+      "epoch": 5.941646973277586,
+      "grad_norm": 0.7939499020576477,
+      "learning_rate": 4.100293811358773e-05,
+      "loss": 0.5466,
+      "mean_token_accuracy": 0.8302853351831436,
+      "num_tokens": 180043519.0,
+      "step": 8175
+    },
+    {
+      "epoch": 5.959825486275223,
+      "grad_norm": 0.840653121471405,
+      "learning_rate": 4.081156707627624e-05,
+      "loss": 0.5432,
+      "mean_token_accuracy": 0.8307925063371658,
+      "num_tokens": 180593050.0,
+      "step": 8200
+    },
+    {
+      "epoch": 5.978003999272859,
+      "grad_norm": 0.8524306416511536,
+      "learning_rate": 4.0620177453204224e-05,
+      "loss": 0.5482,
+      "mean_token_accuracy": 0.8293575036525727,
+      "num_tokens": 181133882.0,
+      "step": 8225
+    },
+    {
+      "epoch": 5.996182512270496,
+      "grad_norm": 0.7460827827453613,
+      "learning_rate": 4.042877362740026e-05,
+      "loss": 0.5553,
+      "mean_token_accuracy": 0.8276761430501938,
+      "num_tokens": 181691715.0,
+      "step": 8250
+    },
+    {
+      "epoch": 5.996182512270496,
+      "eval_loss": 0.5585607886314392,
+      "eval_mean_token_accuracy": 0.8256761994626787,
+      "eval_num_tokens": 181691715.0,
+      "eval_runtime": 112.4662,
+      "eval_samples_per_second": 43.48,
+      "eval_steps_per_second": 5.442,
+      "step": 8250
+    },
+    {
+      "epoch": 6.013815669878204,
+      "grad_norm": 0.8060737252235413,
+      "learning_rate": 4.02373599822182e-05,
+      "loss": 0.5458,
+      "mean_token_accuracy": 0.8301435733578869,
+      "num_tokens": 182221571.0,
+      "step": 8275
+    },
+    {
+      "epoch": 6.031994182875841,
+      "grad_norm": 0.7671223878860474,
+      "learning_rate": 4.004594090123678e-05,
+      "loss": 0.5435,
+      "mean_token_accuracy": 0.8307790219783783,
+      "num_tokens": 182762751.0,
+      "step": 8300
+    },
+    {
+      "epoch": 6.050172695873478,
+      "grad_norm": 0.7601523995399475,
+      "learning_rate": 3.985452076815922e-05,
+      "loss": 0.5353,
+      "mean_token_accuracy": 0.83306546241045,
+      "num_tokens": 183305848.0,
+      "step": 8325
+    },
+    {
+      "epoch": 6.068351208871114,
+      "grad_norm": 0.848628044128418,
+      "learning_rate": 3.966310396671283e-05,
+      "loss": 0.5394,
+      "mean_token_accuracy": 0.8321545705199241,
+      "num_tokens": 183850351.0,
+      "step": 8350
+    },
+    {
+      "epoch": 6.086529721868751,
+      "grad_norm": 0.7746974229812622,
+      "learning_rate": 3.9471694880548625e-05,
+      "loss": 0.5387,
+      "mean_token_accuracy": 0.8319272243976593,
+      "num_tokens": 184403683.0,
+      "step": 8375
+    },
+    {
+      "epoch": 6.104708234866388,
+      "grad_norm": 0.8478145599365234,
+      "learning_rate": 3.9280297893140924e-05,
+      "loss": 0.5316,
+      "mean_token_accuracy": 0.8347961682081223,
+      "num_tokens": 184948375.0,
+      "step": 8400
+    },
+    {
+      "epoch": 6.122886747864024,
+      "grad_norm": 0.7637465000152588,
+      "learning_rate": 3.9088917387686984e-05,
+      "loss": 0.545,
+      "mean_token_accuracy": 0.8306651490926743,
+      "num_tokens": 185499651.0,
+      "step": 8425
+    },
+    {
+      "epoch": 6.141065260861661,
+      "grad_norm": 0.8355098366737366,
+      "learning_rate": 3.8897557747006604e-05,
+      "loss": 0.537,
+      "mean_token_accuracy": 0.8326031097769737,
+      "num_tokens": 186059243.0,
+      "step": 8450
+    },
+    {
+      "epoch": 6.159243773859298,
+      "grad_norm": 0.8073794841766357,
+      "learning_rate": 3.870622335344174e-05,
+      "loss": 0.5374,
+      "mean_token_accuracy": 0.8339161434769631,
+      "num_tokens": 186615115.0,
+      "step": 8475
+    },
+    {
+      "epoch": 6.1774222868569355,
+      "grad_norm": 0.7971922159194946,
+      "learning_rate": 3.851491858875619e-05,
+      "loss": 0.5328,
+      "mean_token_accuracy": 0.8337607860565186,
+      "num_tokens": 187158233.0,
+      "step": 8500
+    },
+    {
+      "epoch": 6.1774222868569355,
+      "eval_loss": 0.5578206181526184,
+      "eval_mean_token_accuracy": 0.8258760601671693,
+      "eval_num_tokens": 187158233.0,
+      "eval_runtime": 112.9076,
+      "eval_samples_per_second": 43.31,
+      "eval_steps_per_second": 5.42,
+      "step": 8500
     }
   ],
   "logging_steps": 25,
@@ -3236,7 +3436,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.4438800024417075e+17,
+  "total_flos": 4.7210511534863155e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null