Training in progress, step 9500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:569ec90df3be5964c1b097a10b54e52c90a472078772ad8e6f9345b6a9133db3
+oid sha256:4258719dcecebc1c0843dab41638ebbcc2b7c072f2a468a899ab44b463281542
 size 12017472

last-checkpoint/global_step9500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87bfb3658af051bd5168f18c84a6c737b4d026357d647dd16623952cf7279a80
+size 71982309

last-checkpoint/global_step9500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc2da02708dd37de2715719da80cd634ddff717ca29c5c0b45229054bf35b33f
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step9000
+global_step9500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1aab311800534a13a9d7022b8b11928b19d22b45728daa1e473cebeb217f764
+oid sha256:800aef772fd8643d6401fb8f1a4a953f0b1f6d395b08356d3c11c489a5ab7481
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 9000,
-  "best_metric": 0.5540527105331421,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-9000",
-  "epoch": 6.540992546809671,
+  "best_global_step": 9500,
+  "best_metric": 0.5516709685325623,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-9500",
+  "epoch": 6.9045628067624065,
   "eval_steps": 250,
-  "global_step": 9000,
+  "global_step": 9500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3617,6 +3617,206 @@
       "eval_samples_per_second": 43.556,
       "eval_steps_per_second": 5.451,
       "step": 9000
+    },
+    {
+      "epoch": 6.559171059807308,
+      "grad_norm": 0.8078477382659912,
+      "learning_rate": 3.4512111433749765e-05,
+      "loss": 0.5355,
+      "mean_token_accuracy": 0.8331593692302703,
+      "num_tokens": 198783104.0,
+      "step": 9025
+    },
+    {
+      "epoch": 6.577349572804945,
+      "grad_norm": 0.7993205189704895,
+      "learning_rate": 3.432256465223894e-05,
+      "loss": 0.5402,
+      "mean_token_accuracy": 0.831983962059021,
+      "num_tokens": 199329947.0,
+      "step": 9050
+    },
+    {
+      "epoch": 6.595528085802581,
+      "grad_norm": 0.8338197469711304,
+      "learning_rate": 3.413314789011529e-05,
+      "loss": 0.5374,
+      "mean_token_accuracy": 0.8321248868107796,
+      "num_tokens": 199881416.0,
+      "step": 9075
+    },
+    {
+      "epoch": 6.613706598800218,
+      "grad_norm": 0.8037586808204651,
+      "learning_rate": 3.394386548522676e-05,
+      "loss": 0.546,
+      "mean_token_accuracy": 0.8305454310774804,
+      "num_tokens": 200430769.0,
+      "step": 9100
+    },
+    {
+      "epoch": 6.631885111797855,
+      "grad_norm": 0.8201845288276672,
+      "learning_rate": 3.375472177234437e-05,
+      "loss": 0.5399,
+      "mean_token_accuracy": 0.8318346044421197,
+      "num_tokens": 200978715.0,
+      "step": 9125
+    },
+    {
+      "epoch": 6.650063624795492,
+      "grad_norm": 0.797753095626831,
+      "learning_rate": 3.356572108306296e-05,
+      "loss": 0.5328,
+      "mean_token_accuracy": 0.8343482685089111,
+      "num_tokens": 201521009.0,
+      "step": 9150
+    },
+    {
+      "epoch": 6.668242137793128,
+      "grad_norm": 0.8221126794815063,
+      "learning_rate": 3.337686774570198e-05,
+      "loss": 0.5374,
+      "mean_token_accuracy": 0.8342596819996834,
+      "num_tokens": 202050884.0,
+      "step": 9175
+    },
+    {
+      "epoch": 6.686420650790765,
+      "grad_norm": 0.7923447489738464,
+      "learning_rate": 3.318816608520636e-05,
+      "loss": 0.5394,
+      "mean_token_accuracy": 0.832169133424759,
+      "num_tokens": 202609530.0,
+      "step": 9200
+    },
+    {
+      "epoch": 6.704599163788402,
+      "grad_norm": 0.8144327402114868,
+      "learning_rate": 3.2999620423047444e-05,
+      "loss": 0.5349,
+      "mean_token_accuracy": 0.8340477573871613,
+      "num_tokens": 203151783.0,
+      "step": 9225
+    },
+    {
+      "epoch": 6.7227776767860385,
+      "grad_norm": 0.808747410774231,
+      "learning_rate": 3.281123507712407e-05,
+      "loss": 0.5435,
+      "mean_token_accuracy": 0.8309207037091255,
+      "num_tokens": 203691329.0,
+      "step": 9250
+    },
+    {
+      "epoch": 6.7227776767860385,
+      "eval_loss": 0.5527983903884888,
+      "eval_mean_token_accuracy": 0.827514678532002,
+      "eval_num_tokens": 203691329.0,
+      "eval_runtime": 112.8236,
+      "eval_samples_per_second": 43.342,
+      "eval_steps_per_second": 5.424,
+      "step": 9250
+    },
+    {
+      "epoch": 6.740956189783676,
+      "grad_norm": 0.7522659301757812,
+      "learning_rate": 3.2623014361663655e-05,
+      "loss": 0.5355,
+      "mean_token_accuracy": 0.8326027810573577,
+      "num_tokens": 204249043.0,
+      "step": 9275
+    },
+    {
+      "epoch": 6.759134702781313,
+      "grad_norm": 0.8386163711547852,
+      "learning_rate": 3.2434962587123394e-05,
+      "loss": 0.5466,
+      "mean_token_accuracy": 0.8312074172496796,
+      "num_tokens": 204801326.0,
+      "step": 9300
+    },
+    {
+      "epoch": 6.77731321577895,
+      "grad_norm": 0.8563323616981506,
+      "learning_rate": 3.2247084060091554e-05,
+      "loss": 0.5401,
+      "mean_token_accuracy": 0.8324404546618461,
+      "num_tokens": 205365857.0,
+      "step": 9325
+    },
+    {
+      "epoch": 6.795491728776586,
+      "grad_norm": 0.8469668030738831,
+      "learning_rate": 3.205938308318887e-05,
+      "loss": 0.5365,
+      "mean_token_accuracy": 0.8328872221708298,
+      "num_tokens": 205919525.0,
+      "step": 9350
+    },
+    {
+      "epoch": 6.813670241774223,
+      "grad_norm": 0.8146055340766907,
+      "learning_rate": 3.187186395496996e-05,
+      "loss": 0.5374,
+      "mean_token_accuracy": 0.8319051578640938,
+      "num_tokens": 206468920.0,
+      "step": 9375
+    },
+    {
+      "epoch": 6.83184875477186,
+      "grad_norm": 0.7568976283073425,
+      "learning_rate": 3.1684530969824895e-05,
+      "loss": 0.5285,
+      "mean_token_accuracy": 0.8365351390838623,
+      "num_tokens": 207011308.0,
+      "step": 9400
+    },
+    {
+      "epoch": 6.850027267769496,
+      "grad_norm": 0.7712914943695068,
+      "learning_rate": 3.1497388417880935e-05,
+      "loss": 0.5426,
+      "mean_token_accuracy": 0.8314177936315537,
+      "num_tokens": 207557360.0,
+      "step": 9425
+    },
+    {
+      "epoch": 6.868205780767133,
+      "grad_norm": 0.8087723851203918,
+      "learning_rate": 3.131044058490415e-05,
+      "loss": 0.5428,
+      "mean_token_accuracy": 0.8317897391319274,
+      "num_tokens": 208118474.0,
+      "step": 9450
+    },
+    {
+      "epoch": 6.88638429376477,
+      "grad_norm": 0.8176565766334534,
+      "learning_rate": 3.112369175220138e-05,
+      "loss": 0.5455,
+      "mean_token_accuracy": 0.8312569990754127,
+      "num_tokens": 208649698.0,
+      "step": 9475
+    },
+    {
+      "epoch": 6.9045628067624065,
+      "grad_norm": 0.8005684614181519,
+      "learning_rate": 3.093714619652211e-05,
+      "loss": 0.5438,
+      "mean_token_accuracy": 0.8314751309156417,
+      "num_tokens": 209201375.0,
+      "step": 9500
+    },
+    {
+      "epoch": 6.9045628067624065,
+      "eval_loss": 0.5516709685325623,
+      "eval_mean_token_accuracy": 0.8279460390019261,
+      "eval_num_tokens": 209201375.0,
+      "eval_runtime": 114.1031,
+      "eval_samples_per_second": 42.856,
+      "eval_steps_per_second": 5.364,
+      "step": 9500
     }
   ],
   "logging_steps": 25,
@@ -3636,7 +3836,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.9990258137235456e+17,
+  "total_flos": 5.276420653375816e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null