Training in progress, step 12000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d739a46be07afc08058bcee6abb1772a84e044deaf39817666f3049bcf653c23
+oid sha256:d553c2e399ec3b829abae04cec3b1f8d3912f3fbed3f20293d8080f079fd2384
 size 12017472

last-checkpoint/global_step12000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ca0684aa48abff7305a3993c60350cc16307d590903d35bab2763d93196f7ec
+size 71982309

last-checkpoint/global_step12000/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:458a0f2126d2e651f4b6c3bae0c3c265ebcf9adcd4b370723bb56b05fd8a7734
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step11500
+global_step12000

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d17e6956d333adf450e550fb2bbfe82bc47be67acb5350845a13faa81c890b40
+oid sha256:cfa8ca0fa51002cbd049096e980f791791ccf3dd111ae5e6ddeefe7a21364f2d
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 11500,
-  "best_metric": 0.544745683670044,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-11500",
-  "epoch": 8.357753135793493,
+  "best_global_step": 12000,
+  "best_metric": 0.5422044396400452,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-12000",
+  "epoch": 8.721323395746229,
   "eval_steps": 250,
-  "global_step": 11500,
+  "global_step": 12000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4617,6 +4617,206 @@
       "eval_samples_per_second": 42.862,
       "eval_steps_per_second": 5.364,
       "step": 11500
+    },
+    {
+      "epoch": 8.375931648791129,
+      "grad_norm": 0.799972414970398,
+      "learning_rate": 1.6883042740215607e-05,
+      "loss": 0.5202,
+      "mean_token_accuracy": 0.8381973811984063,
+      "num_tokens": 253842951.0,
+      "step": 11525
+    },
+    {
+      "epoch": 8.394110161788765,
+      "grad_norm": 0.8211126327514648,
+      "learning_rate": 1.6727091589575087e-05,
+      "loss": 0.5242,
+      "mean_token_accuracy": 0.8364318865537643,
+      "num_tokens": 254394355.0,
+      "step": 11550
+    },
+    {
+      "epoch": 8.412288674786403,
+      "grad_norm": 0.8928632140159607,
+      "learning_rate": 1.6571673413614786e-05,
+      "loss": 0.5203,
+      "mean_token_accuracy": 0.8383208379149437,
+      "num_tokens": 254928725.0,
+      "step": 11575
+    },
+    {
+      "epoch": 8.43046718778404,
+      "grad_norm": 0.8527329564094543,
+      "learning_rate": 1.641679177157841e-05,
+      "loss": 0.5275,
+      "mean_token_accuracy": 0.8369225415587426,
+      "num_tokens": 255482197.0,
+      "step": 11600
+    },
+    {
+      "epoch": 8.448645700781675,
+      "grad_norm": 0.8391927480697632,
+      "learning_rate": 1.626245021042244e-05,
+      "loss": 0.5318,
+      "mean_token_accuracy": 0.8345513901114464,
+      "num_tokens": 256040695.0,
+      "step": 11625
+    },
+    {
+      "epoch": 8.466824213779313,
+      "grad_norm": 0.816584587097168,
+      "learning_rate": 1.6108652264734953e-05,
+      "loss": 0.5231,
+      "mean_token_accuracy": 0.8369111514091492,
+      "num_tokens": 256592914.0,
+      "step": 11650
+    },
+    {
+      "epoch": 8.48500272677695,
+      "grad_norm": 0.8552756309509277,
+      "learning_rate": 1.5955401456654614e-05,
+      "loss": 0.5156,
+      "mean_token_accuracy": 0.8394366270303726,
+      "num_tokens": 257129982.0,
+      "step": 11675
+    },
+    {
+      "epoch": 8.503181239774586,
+      "grad_norm": 0.9166038632392883,
+      "learning_rate": 1.5802701295790058e-05,
+      "loss": 0.5289,
+      "mean_token_accuracy": 0.8356350338459015,
+      "num_tokens": 257674901.0,
+      "step": 11700
+    },
+    {
+      "epoch": 8.521359752772224,
+      "grad_norm": 0.8325772285461426,
+      "learning_rate": 1.565055527913954e-05,
+      "loss": 0.5295,
+      "mean_token_accuracy": 0.8354543587565422,
+      "num_tokens": 258229868.0,
+      "step": 11725
+    },
+    {
+      "epoch": 8.53953826576986,
+      "grad_norm": 0.8623970150947571,
+      "learning_rate": 1.5498966891010768e-05,
+      "loss": 0.5158,
+      "mean_token_accuracy": 0.8386522510647774,
+      "num_tokens": 258782034.0,
+      "step": 11750
+    },
+    {
+      "epoch": 8.53953826576986,
+      "eval_loss": 0.5427566170692444,
+      "eval_mean_token_accuracy": 0.8309165983418234,
+      "eval_num_tokens": 258782034.0,
+      "eval_runtime": 114.0537,
+      "eval_samples_per_second": 42.875,
+      "eval_steps_per_second": 5.366,
+      "step": 11750
+    },
+    {
+      "epoch": 8.557716778767496,
+      "grad_norm": 0.7980916500091553,
+      "learning_rate": 1.5347939602941168e-05,
+      "loss": 0.5201,
+      "mean_token_accuracy": 0.8383438029885292,
+      "num_tokens": 259333447.0,
+      "step": 11775
+    },
+    {
+      "epoch": 8.575895291765134,
+      "grad_norm": 0.8518940806388855,
+      "learning_rate": 1.5197476873618385e-05,
+      "loss": 0.5208,
+      "mean_token_accuracy": 0.8381571528315545,
+      "num_tokens": 259880015.0,
+      "step": 11800
+    },
+    {
+      "epoch": 8.59407380476277,
+      "grad_norm": 0.8770802617073059,
+      "learning_rate": 1.5047582148801043e-05,
+      "loss": 0.5159,
+      "mean_token_accuracy": 0.8389484801888466,
+      "num_tokens": 260448319.0,
+      "step": 11825
+    },
+    {
+      "epoch": 8.612252317760408,
+      "grad_norm": 0.8422465324401855,
+      "learning_rate": 1.489825886123987e-05,
+      "loss": 0.5149,
+      "mean_token_accuracy": 0.8405806881189346,
+      "num_tokens": 260995823.0,
+      "step": 11850
+    },
+    {
+      "epoch": 8.630430830758044,
+      "grad_norm": 0.8910753726959229,
+      "learning_rate": 1.4749510430599028e-05,
+      "loss": 0.5178,
+      "mean_token_accuracy": 0.838139820098877,
+      "num_tokens": 261537207.0,
+      "step": 11875
+    },
+    {
+      "epoch": 8.64860934375568,
+      "grad_norm": 0.7846816182136536,
+      "learning_rate": 1.460134026337789e-05,
+      "loss": 0.518,
+      "mean_token_accuracy": 0.8382885718345642,
+      "num_tokens": 262095255.0,
+      "step": 11900
+    },
+    {
+      "epoch": 8.666787856753318,
+      "grad_norm": 0.8034218549728394,
+      "learning_rate": 1.445375175283294e-05,
+      "loss": 0.5263,
+      "mean_token_accuracy": 0.8366366970539093,
+      "num_tokens": 262638805.0,
+      "step": 11925
+    },
+    {
+      "epoch": 8.684966369750954,
+      "grad_norm": 0.8177446126937866,
+      "learning_rate": 1.4306748278900102e-05,
+      "loss": 0.5208,
+      "mean_token_accuracy": 0.8381500113010406,
+      "num_tokens": 263176118.0,
+      "step": 11950
+    },
+    {
+      "epoch": 8.70314488274859,
+      "grad_norm": 0.890275239944458,
+      "learning_rate": 1.4160333208117326e-05,
+      "loss": 0.5181,
+      "mean_token_accuracy": 0.8382448080182076,
+      "num_tokens": 263729198.0,
+      "step": 11975
+    },
+    {
+      "epoch": 8.721323395746229,
+      "grad_norm": 0.850255012512207,
+      "learning_rate": 1.4014509893547503e-05,
+      "loss": 0.5279,
+      "mean_token_accuracy": 0.835902649462223,
+      "num_tokens": 264289934.0,
+      "step": 12000
+    },
+    {
+      "epoch": 8.721323395746229,
+      "eval_loss": 0.5422044396400452,
+      "eval_mean_token_accuracy": 0.8310892993912977,
+      "eval_num_tokens": 264289934.0,
+      "eval_runtime": 113.7092,
+      "eval_samples_per_second": 43.004,
+      "eval_steps_per_second": 5.382,
+      "step": 12000
     }
   ],
   "logging_steps": 25,
@@ -4636,7 +4836,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.387220694035333e+17,
+  "total_flos": 6.664963569651548e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null