Training in progress, step 5500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81eac720b158c7f43a3b9b48f3c680e3548bab4820189790d8de2f257ac92036
+oid sha256:e175cfbb3b1e5047d2a07a1f65e6011d48b21f6ac86f4b54bb7a003b3e25ddd9
 size 12017472

last-checkpoint/global_step5500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1260828109bb51d4ec084d9edc433ecc1fd867fbe5d156a6c5ea7d7b2fe0528
+size 71982309

last-checkpoint/global_step5500/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ec0cdef2fdf8552c3a2888f93dae3f9ed787dcab7d4e98334b4826387d6456a
+size 146356645

last-checkpoint/latest

@@ -1 +1 @@
-global_step5000
+global_step5500

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3dd594d08139e0846701d4c186ee22eb3ed05631cdda05ef04a8843616048835
+oid sha256:c75d63279b47e795ad4622a2e3404a0983cd22c3e120a053ebabf9c78e50af21
 size 14709

last-checkpoint/trainer_state.json

@@ -1,10 +1,10 @@
 {
-  "best_global_step": 5000,
-  "best_metric": 0.5900602340698242,
-  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-5000",
-  "epoch": 3.6340665333575712,
+  "best_global_step": 5500,
+  "best_metric": 0.5829094648361206,
+  "best_model_checkpoint": "/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep/checkpoint-5500",
+  "epoch": 3.997636793310307,
   "eval_steps": 250,
-  "global_step": 5000,
+  "global_step": 5500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2017,6 +2017,206 @@
       "eval_samples_per_second": 43.688,
       "eval_steps_per_second": 5.468,
       "step": 5000
+    },
+    {
+      "epoch": 3.6522450463552083,
+      "grad_norm": 0.7423418760299683,
+      "learning_rate": 6.350273695266381e-05,
+      "loss": 0.5796,
+      "mean_token_accuracy": 0.82060361713171,
+      "num_tokens": 110631489.0,
+      "step": 5025
+    },
+    {
+      "epoch": 3.670423559352845,
+      "grad_norm": 0.8051418662071228,
+      "learning_rate": 6.334757586954799e-05,
+      "loss": 0.5858,
+      "mean_token_accuracy": 0.818088726401329,
+      "num_tokens": 111178589.0,
+      "step": 5050
+    },
+    {
+      "epoch": 3.6886020723504815,
+      "grad_norm": 0.8583251237869263,
+      "learning_rate": 6.319188010178665e-05,
+      "loss": 0.5859,
+      "mean_token_accuracy": 0.818018836081028,
+      "num_tokens": 111748170.0,
+      "step": 5075
+    },
+    {
+      "epoch": 3.7067805853481186,
+      "grad_norm": 0.719312846660614,
+      "learning_rate": 6.303565321498068e-05,
+      "loss": 0.585,
+      "mean_token_accuracy": 0.8183588898181915,
+      "num_tokens": 112291726.0,
+      "step": 5100
+    },
+    {
+      "epoch": 3.7249590983457552,
+      "grad_norm": 0.768618106842041,
+      "learning_rate": 6.287889878689412e-05,
+      "loss": 0.5833,
+      "mean_token_accuracy": 0.8190835517644882,
+      "num_tokens": 112838799.0,
+      "step": 5125
+    },
+    {
+      "epoch": 3.743137611343392,
+      "grad_norm": 0.8095366358757019,
+      "learning_rate": 6.272162040737227e-05,
+      "loss": 0.586,
+      "mean_token_accuracy": 0.8185149121284485,
+      "num_tokens": 113391677.0,
+      "step": 5150
+    },
+    {
+      "epoch": 3.761316124341029,
+      "grad_norm": 0.762540340423584,
+      "learning_rate": 6.256382167825952e-05,
+      "loss": 0.5816,
+      "mean_token_accuracy": 0.8203315672278404,
+      "num_tokens": 113936197.0,
+      "step": 5175
+    },
+    {
+      "epoch": 3.779494637338666,
+      "grad_norm": 0.7430989146232605,
+      "learning_rate": 6.240550621331678e-05,
+      "loss": 0.5821,
+      "mean_token_accuracy": 0.8193337711691856,
+      "num_tokens": 114493484.0,
+      "step": 5200
+    },
+    {
+      "epoch": 3.7976731503363026,
+      "grad_norm": 0.7766333222389221,
+      "learning_rate": 6.224667763813883e-05,
+      "loss": 0.5833,
+      "mean_token_accuracy": 0.8199924173951149,
+      "num_tokens": 115036265.0,
+      "step": 5225
+    },
+    {
+      "epoch": 3.815851663333939,
+      "grad_norm": 0.7837154269218445,
+      "learning_rate": 6.208733959007113e-05,
+      "loss": 0.5743,
+      "mean_token_accuracy": 0.8223940685391427,
+      "num_tokens": 115580134.0,
+      "step": 5250
+    },
+    {
+      "epoch": 3.815851663333939,
+      "eval_loss": 0.5864232778549194,
+      "eval_mean_token_accuracy": 0.8171391937078214,
+      "eval_num_tokens": 115580134.0,
+      "eval_runtime": 112.2751,
+      "eval_samples_per_second": 43.554,
+      "eval_steps_per_second": 5.451,
+      "step": 5250
+    },
+    {
+      "epoch": 3.8340301763315763,
+      "grad_norm": 0.7619733810424805,
+      "learning_rate": 6.19274957181267e-05,
+      "loss": 0.5814,
+      "mean_token_accuracy": 0.8204685914516449,
+      "num_tokens": 116138332.0,
+      "step": 5275
+    },
+    {
+      "epoch": 3.852208689329213,
+      "grad_norm": 0.7965566515922546,
+      "learning_rate": 6.176714968290246e-05,
+      "loss": 0.589,
+      "mean_token_accuracy": 0.8183123478293419,
+      "num_tokens": 116694993.0,
+      "step": 5300
+    },
+    {
+      "epoch": 3.8703872023268495,
+      "grad_norm": 0.7417636513710022,
+      "learning_rate": 6.160630515649538e-05,
+      "loss": 0.5843,
+      "mean_token_accuracy": 0.8171842768788338,
+      "num_tokens": 117258648.0,
+      "step": 5325
+    },
+    {
+      "epoch": 3.8885657153244866,
+      "grad_norm": 0.7932421565055847,
+      "learning_rate": 6.144496582241842e-05,
+      "loss": 0.5811,
+      "mean_token_accuracy": 0.819763533771038,
+      "num_tokens": 117811709.0,
+      "step": 5350
+    },
+    {
+      "epoch": 3.906744228322123,
+      "grad_norm": 0.7742260694503784,
+      "learning_rate": 6.128313537551622e-05,
+      "loss": 0.583,
+      "mean_token_accuracy": 0.8193339914083481,
+      "num_tokens": 118368306.0,
+      "step": 5375
+    },
+    {
+      "epoch": 3.92492274131976,
+      "grad_norm": 0.7565304040908813,
+      "learning_rate": 6.112081752188036e-05,
+      "loss": 0.5913,
+      "mean_token_accuracy": 0.816694650053978,
+      "num_tokens": 118923425.0,
+      "step": 5400
+    },
+    {
+      "epoch": 3.943101254317397,
+      "grad_norm": 0.733914315700531,
+      "learning_rate": 6.09580159787646e-05,
+      "loss": 0.5797,
+      "mean_token_accuracy": 0.8201961496472359,
+      "num_tokens": 119479832.0,
+      "step": 5425
+    },
+    {
+      "epoch": 3.9612797673150335,
+      "grad_norm": 0.800318717956543,
+      "learning_rate": 6.07947344744997e-05,
+      "loss": 0.5798,
+      "mean_token_accuracy": 0.8196702027320861,
+      "num_tokens": 120025792.0,
+      "step": 5450
+    },
+    {
+      "epoch": 3.9794582803126706,
+      "grad_norm": 0.7655355334281921,
+      "learning_rate": 6.0630976748408074e-05,
+      "loss": 0.5763,
+      "mean_token_accuracy": 0.8210220813751221,
+      "num_tokens": 120587031.0,
+      "step": 5475
+    },
+    {
+      "epoch": 3.997636793310307,
+      "grad_norm": 0.8047225475311279,
+      "learning_rate": 6.046674655071809e-05,
+      "loss": 0.573,
+      "mean_token_accuracy": 0.8227437067031861,
+      "num_tokens": 121138768.0,
+      "step": 5500
+    },
+    {
+      "epoch": 3.997636793310307,
+      "eval_loss": 0.5829094648361206,
+      "eval_mean_token_accuracy": 0.8182347503557704,
+      "eval_num_tokens": 121138768.0,
+      "eval_runtime": 112.4332,
+      "eval_samples_per_second": 43.492,
+      "eval_steps_per_second": 5.443,
+      "step": 5500
     }
   ],
   "logging_steps": 25,
@@ -2036,7 +2236,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.7767572610102067e+17,
+  "total_flos": 3.0548540768282214e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null